diff --git a/pkg/frontend/authenticate.go b/pkg/frontend/authenticate.go index e8b6539fc629f..53edf3cc17e20 100644 --- a/pkg/frontend/authenticate.go +++ b/pkg/frontend/authenticate.go @@ -1458,10 +1458,6 @@ const ( checkStageFormat = `select stage_id, stage_name from mo_catalog.mo_stages where stage_name = "%s" order by stage_id;` - checkStageStatusFormat = `select stage_id, stage_name from mo_catalog.mo_stages where stage_status = "%s" order by stage_id;` - - checkStageStatusWithStageNameFormat = `select url, stage_status from mo_catalog.mo_stages where stage_name = "%s" order by stage_id;` - dropStageFormat = `delete from mo_catalog.mo_stages where stage_name = '%s' order by stage_id;` updateStageUrlFormat = `update mo_catalog.mo_stages set url = '%s' where stage_name = '%s' order by stage_id;` @@ -1538,20 +1534,9 @@ func getSqlForCheckStage(ctx context.Context, stage string) (string, error) { return fmt.Sprintf(checkStageFormat, stage), nil } -func getSqlForCheckStageStatus(ctx context.Context, status string) string { - return fmt.Sprintf(checkStageStatusFormat, status) -} - func getSqlForCheckUdfWithDb(dbName string) string { return fmt.Sprintf(checkUdfWithDb, dbName) } -func getSqlForCheckStageStatusWithStageName(ctx context.Context, stage string) (string, error) { - err := inputNameIsInvalid(ctx, stage) - if err != nil { - return "", err - } - return fmt.Sprintf(checkStageStatusWithStageNameFormat, stage), nil -} func getSqlForInsertIntoMoStages(ctx context.Context, stageName, url, credentials, status, createdTime, comment string) (string, error) { err := inputNameIsInvalid(ctx, stageName) @@ -3266,7 +3251,7 @@ func doCreateStage(ctx context.Context, ses *Session, cs *tree.CreateStage) (err } } else { // format credentials and hash it - credentials = HashPassWord(formatCredentials(cs.Credentials)) + credentials = formatCredentials(cs.Credentials) if !cs.Status.Exist { StageStatus = "disabled" @@ -3278,6 +3263,11 @@ func doCreateStage(ctx context.Context, ses *Session, cs *tree.CreateStage) (err comment = cs.Comment.Comment } + if !(strings.HasPrefix(cs.Url, function.STAGE_PROTOCOL+"://") || strings.HasPrefix(cs.Url, function.S3_PROTOCOL+"://") || + strings.HasPrefix(cs.Url, function.FILE_PROTOCOL+":///")) { + return moerr.NewBadConfig(ctx, "URL protocol only supports stage://, s3:// and file:///") + } + sql, err = getSqlForInsertIntoMoStages(ctx, string(cs.Name), cs.Url, credentials, StageStatus, types.CurrentTimestamp().String2(time.UTC, 0), comment) if err != nil { return err @@ -3295,88 +3285,24 @@ func doCreateStage(ctx context.Context, ses *Session, cs *tree.CreateStage) (err func doCheckFilePath(ctx context.Context, ses *Session, ep *tree.ExportParam) (err error) { //var err error var filePath string - var sql string - var erArray []ExecResult - var stageName string - var stageStatus string - var url string if ep == nil { return err } - bh := ses.GetBackgroundExec(ctx) - defer bh.Close() - - err = bh.Exec(ctx, "begin;") - defer func() { - err = finishTxn(ctx, bh, err) - }() - if err != nil { - return err - } - // detect filepath contain stage or not filePath = ep.FilePath - if !strings.Contains(filePath, ":") { - // the filepath is the target path - sql = getSqlForCheckStageStatus(ctx, "enabled") - bh.ClearExecResultSet() - err = bh.Exec(ctx, sql) + if strings.HasPrefix(filePath, function.STAGE_PROTOCOL+"://") { + // stage:// URL + s, err := function.UrlToStageDef(filePath, ses.proc) if err != nil { return err } - erArray, err = getResultSet(ctx, bh) + // s.ToPath() returns the fileservice filepath, i.e. s3,...:/path for S3 or /path for local file + ses.ep.userConfig.StageFilePath, _, err = s.ToPath() if err != nil { return err } - - // if have stage enabled - if execResultArrayHasData(erArray) { - return moerr.NewInternalError(ctx, "stage exists, please try to check and use a stage instead") - } else { - // use the filepath - return err - } - } else { - stageName = strings.Split(filePath, ":")[0] - // check the stage status - sql, err = getSqlForCheckStageStatusWithStageName(ctx, stageName) - if err != nil { - return err - } - bh.ClearExecResultSet() - err = bh.Exec(ctx, sql) - if err != nil { - return err - } - - erArray, err = getResultSet(ctx, bh) - if err != nil { - return err - } - if execResultArrayHasData(erArray) { - stageStatus, err = erArray[0].GetString(ctx, 0, 1) - if err != nil { - return err - } - - // is the stage staus is disabled - if stageStatus == tree.StageStatusDisabled.String() { - return moerr.NewInternalError(ctx, "stage '%s' is invalid, please check", stageName) - } else if stageStatus == tree.StageStatusEnabled.String() { - // replace the filepath using stage url - url, err = erArray[0].GetString(ctx, 0, 0) - if err != nil { - return err - } - - filePath = strings.Replace(filePath, stageName+":", url, 1) - ses.ep.userConfig.StageFilePath = filePath - } - } else { - return moerr.NewInternalError(ctx, "stage '%s' is not exists, please check", stageName) - } } return err @@ -3440,6 +3366,11 @@ func doAlterStage(ctx context.Context, ses *Session, as *tree.AlterStage) (err e } } else { if as.UrlOption.Exist { + if !(strings.HasPrefix(as.UrlOption.Url, function.STAGE_PROTOCOL+"://") || + strings.HasPrefix(as.UrlOption.Url, function.S3_PROTOCOL+"://") || + strings.HasPrefix(as.UrlOption.Url, function.FILE_PROTOCOL+":///")) { + return moerr.NewBadConfig(ctx, "URL protocol only supports stage://, s3:// and file:///") + } sql = getsqlForUpdateStageUrl(string(as.Name), as.UrlOption.Url) err = bh.Exec(ctx, sql) if err != nil { @@ -3448,7 +3379,7 @@ func doAlterStage(ctx context.Context, ses *Session, as *tree.AlterStage) (err e } if as.CredentialsOption.Exist { - credentials = HashPassWord(formatCredentials(as.CredentialsOption)) + credentials = formatCredentials(as.CredentialsOption) sql = getsqlForUpdateStageCredentials(string(as.Name), credentials) err = bh.Exec(ctx, sql) if err != nil { diff --git a/pkg/frontend/authenticate_test.go b/pkg/frontend/authenticate_test.go index 214296bfb67c3..4fae905907d74 100644 --- a/pkg/frontend/authenticate_test.go +++ b/pkg/frontend/authenticate_test.go @@ -9603,7 +9603,7 @@ func TestDoCreateStage(t *testing.T) { cs := &tree.CreateStage{ IfNotExists: false, Name: tree.Identifier("my_stage_test"), - Url: "'s3://load/files/'", + Url: "s3://load/files/", Credentials: tree.StageCredentials{ Exist: false, }, @@ -9661,7 +9661,7 @@ func TestDoCreateStage(t *testing.T) { cs := &tree.CreateStage{ IfNotExists: false, Name: tree.Identifier("my_stage_test"), - Url: "'s3://load/files/'", + Url: "s3://load/files/", Credentials: tree.StageCredentials{ Exist: false, }, @@ -9721,7 +9721,7 @@ func TestDoCreateStage(t *testing.T) { cs := &tree.CreateStage{ IfNotExists: true, Name: tree.Identifier("my_stage_test"), - Url: "'s3://load/files/'", + Url: "file:///load/files/", Credentials: tree.StageCredentials{ Exist: false, }, @@ -9779,10 +9779,10 @@ func TestDoCreateStage(t *testing.T) { cs := &tree.CreateStage{ IfNotExists: false, Name: tree.Identifier("my_stage_test"), - Url: "'s3://load/files/'", + Url: "s3://load/files/", Credentials: tree.StageCredentials{ Exist: true, - Credentials: []string{"'AWS_KEY_ID'", "'1a2b3c'", "'AWS_SECRET_KEY'", "'4x5y6z'"}, + Credentials: []string{"AWS_KEY_ID", "1a2b3c", "AWS_SECRET_KEY", "4x5y6z"}, }, Status: tree.StageStatus{ Exist: true, @@ -9839,7 +9839,7 @@ func TestDoCreateStage(t *testing.T) { cs := &tree.CreateStage{ IfNotExists: false, Name: tree.Identifier("my_stage_test"), - Url: "'s3://load/files/'", + Url: "s3://load/files/", Credentials: tree.StageCredentials{ Exist: false, }, @@ -9901,7 +9901,7 @@ func TestDoAlterStage(t *testing.T) { Name: tree.Identifier("my_stage_test"), UrlOption: tree.StageUrl{ Exist: true, - Url: "'s3://load/files/'", + Url: "s3://load/files/", }, CredentialsOption: tree.StageCredentials{ Exist: false, @@ -9967,7 +9967,7 @@ func TestDoAlterStage(t *testing.T) { }, CredentialsOption: tree.StageCredentials{ Exist: true, - Credentials: []string{"'AWS_KEY_ID'", "'1a2b3c'", "'AWS_SECRET_KEY'", "'4x5y6z'"}, + Credentials: []string{"AWS_KEY_ID", "1a2b3c", "AWS_SECRET_KEY", "4x5y6z"}, }, StatusOption: tree.StageStatus{ Exist: false, @@ -10027,7 +10027,7 @@ func TestDoAlterStage(t *testing.T) { Name: tree.Identifier("my_stage_test"), UrlOption: tree.StageUrl{ Exist: true, - Url: "'s3://load/files/'", + Url: "s3://load/files/", }, CredentialsOption: tree.StageCredentials{ Exist: false, @@ -10088,7 +10088,7 @@ func TestDoAlterStage(t *testing.T) { Name: tree.Identifier("my_stage_test"), UrlOption: tree.StageUrl{ Exist: true, - Url: "'s3://load/files/'", + Url: "s3://load/files/", }, CredentialsOption: tree.StageCredentials{ Exist: false, @@ -10149,11 +10149,11 @@ func TestDoAlterStage(t *testing.T) { Name: tree.Identifier("my_stage_test"), UrlOption: tree.StageUrl{ Exist: true, - Url: "'s3://load/files/'", + Url: "s3://load/files/", }, CredentialsOption: tree.StageCredentials{ Exist: true, - Credentials: []string{"'AWS_KEY_ID'", "'1a2b3c'", "'AWS_SECRET_KEY'", "'4x5y6z'"}, + Credentials: []string{"AWS_KEY_ID", "1a2b3c", "AWS_SECRET_KEY", "4x5y6z"}, }, StatusOption: tree.StageStatus{ Exist: false, @@ -10214,7 +10214,7 @@ func TestDoAlterStage(t *testing.T) { }, CredentialsOption: tree.StageCredentials{ Exist: true, - Credentials: []string{"'AWS_KEY_ID'", "'1a2b3c'", "'AWS_SECRET_KEY'", "'4x5y6z'"}, + Credentials: []string{"AWS_KEY_ID", "1a2b3c", "AWS_SECRET_KEY", "4x5y6z"}, }, StatusOption: tree.StageStatus{ Exist: false, @@ -10241,307 +10241,6 @@ func TestDoAlterStage(t *testing.T) { } -func TestDoCheckFilePath(t *testing.T) { - convey.Convey("doCheckFilePath success", t, func() { - ctrl := gomock.NewController(t) - defer ctrl.Finish() - - ses := newTestSession(t, ctrl) - defer ses.Close() - - bh := &backgroundExecTest{} - bh.init() - - bhStub := gostub.StubFunc(&NewBackgroundExec, bh) - defer bhStub.Reset() - - pu := config.NewParameterUnit(&config.FrontendParameters{}, nil, nil, nil) - pu.SV.SetDefaultValues() - ctx := context.WithValue(context.TODO(), config.ParameterUnitKey, pu) - - rm, _ := NewRoutineManager(ctx) - ses.rm = rm - - tenant := &TenantInfo{ - Tenant: sysAccountName, - User: rootName, - DefaultRole: moAdminRoleName, - TenantID: sysAccountID, - UserID: rootID, - DefaultRoleID: moAdminRoleID, - } - ses.SetTenantInfo(tenant) - - cs := &tree.Select{} - ses.InitExportConfig(cs.Ep) - - //no result set - bh.sql2result["begin;"] = nil - bh.sql2result["commit;"] = nil - bh.sql2result["rollback;"] = nil - - err := doCheckFilePath(ctx, ses, cs.Ep) - convey.So(err, convey.ShouldBeNil) - }) - - convey.Convey("doCheckFilePath success", t, func() { - ctrl := gomock.NewController(t) - defer ctrl.Finish() - - ses := newTestSession(t, ctrl) - defer ses.Close() - - bh := &backgroundExecTest{} - bh.init() - - bhStub := gostub.StubFunc(&NewBackgroundExec, bh) - defer bhStub.Reset() - - pu := config.NewParameterUnit(&config.FrontendParameters{}, nil, nil, nil) - pu.SV.SetDefaultValues() - ctx := context.WithValue(context.TODO(), config.ParameterUnitKey, pu) - - rm, _ := NewRoutineManager(ctx) - ses.rm = rm - - tenant := &TenantInfo{ - Tenant: sysAccountName, - User: rootName, - DefaultRole: moAdminRoleName, - TenantID: sysAccountID, - UserID: rootID, - DefaultRoleID: moAdminRoleID, - } - ses.SetTenantInfo(tenant) - - cs := &tree.Select{ - Ep: &tree.ExportParam{ - FilePath: "/mnt/disk1/t1.csv", - }, - } - ses.InitExportConfig(cs.Ep) - - //no result set - bh.sql2result["begin;"] = nil - bh.sql2result["commit;"] = nil - bh.sql2result["rollback;"] = nil - - sql := getSqlForCheckStageStatus(ctx, "enabled") - mrs := newMrsForPasswordOfUser([][]interface{}{}) - bh.sql2result[sql] = mrs - - err := doCheckFilePath(ctx, ses, cs.Ep) - convey.So(err, convey.ShouldBeNil) - }) - - convey.Convey("doCheckFilePath success", t, func() { - ctrl := gomock.NewController(t) - defer ctrl.Finish() - - ses := newTestSession(t, ctrl) - defer ses.Close() - - bh := &backgroundExecTest{} - bh.init() - - bhStub := gostub.StubFunc(&NewBackgroundExec, bh) - defer bhStub.Reset() - - pu := config.NewParameterUnit(&config.FrontendParameters{}, nil, nil, nil) - pu.SV.SetDefaultValues() - ctx := context.WithValue(context.TODO(), config.ParameterUnitKey, pu) - - rm, _ := NewRoutineManager(ctx) - ses.rm = rm - - tenant := &TenantInfo{ - Tenant: sysAccountName, - User: rootName, - DefaultRole: moAdminRoleName, - TenantID: sysAccountID, - UserID: rootID, - DefaultRoleID: moAdminRoleID, - } - ses.SetTenantInfo(tenant) - - cs := &tree.Select{ - Ep: &tree.ExportParam{ - FilePath: "/mnt/disk1/t1.csv", - }, - } - ses.InitExportConfig(cs.Ep) - - //no result set - bh.sql2result["begin;"] = nil - bh.sql2result["commit;"] = nil - bh.sql2result["rollback;"] = nil - - sql := getSqlForCheckStageStatus(ctx, "enabled") - mrs := newMrsForPasswordOfUser([][]interface{}{ - {0, 0}, - }) - bh.sql2result[sql] = mrs - - err := doCheckFilePath(ctx, ses, cs.Ep) - convey.So(err, convey.ShouldNotBeNil) - }) - - convey.Convey("doCheckFilePath fail", t, func() { - ctrl := gomock.NewController(t) - defer ctrl.Finish() - - ses := newTestSession(t, ctrl) - defer ses.Close() - - bh := &backgroundExecTest{} - bh.init() - - bhStub := gostub.StubFunc(&NewBackgroundExec, bh) - defer bhStub.Reset() - - pu := config.NewParameterUnit(&config.FrontendParameters{}, nil, nil, nil) - pu.SV.SetDefaultValues() - ctx := context.WithValue(context.TODO(), config.ParameterUnitKey, pu) - - rm, _ := NewRoutineManager(ctx) - ses.rm = rm - - tenant := &TenantInfo{ - Tenant: sysAccountName, - User: rootName, - DefaultRole: moAdminRoleName, - TenantID: sysAccountID, - UserID: rootID, - DefaultRoleID: moAdminRoleID, - } - ses.SetTenantInfo(tenant) - - cs := &tree.Select{ - Ep: &tree.ExportParam{ - FilePath: "stage1:/t1.csv", - }, - } - ses.InitExportConfig(cs.Ep) - - //no result set - bh.sql2result["begin;"] = nil - bh.sql2result["commit;"] = nil - bh.sql2result["rollback;"] = nil - - sql, _ := getSqlForCheckStageStatusWithStageName(ctx, "stage1") - mrs := newMrsForPasswordOfUser([][]interface{}{}) - bh.sql2result[sql] = mrs - - err := doCheckFilePath(ctx, ses, cs.Ep) - convey.So(err, convey.ShouldNotBeNil) - }) - - convey.Convey("doCheckFilePath fail", t, func() { - ctrl := gomock.NewController(t) - defer ctrl.Finish() - - ses := newTestSession(t, ctrl) - defer ses.Close() - - bh := &backgroundExecTest{} - bh.init() - - bhStub := gostub.StubFunc(&NewBackgroundExec, bh) - defer bhStub.Reset() - - pu := config.NewParameterUnit(&config.FrontendParameters{}, nil, nil, nil) - pu.SV.SetDefaultValues() - ctx := context.WithValue(context.TODO(), config.ParameterUnitKey, pu) - - rm, _ := NewRoutineManager(ctx) - ses.rm = rm - - tenant := &TenantInfo{ - Tenant: sysAccountName, - User: rootName, - DefaultRole: moAdminRoleName, - TenantID: sysAccountID, - UserID: rootID, - DefaultRoleID: moAdminRoleID, - } - ses.SetTenantInfo(tenant) - - cs := &tree.Select{ - Ep: &tree.ExportParam{ - FilePath: "stage1:/t1.csv", - }, - } - ses.InitExportConfig(cs.Ep) - - //no result set - bh.sql2result["begin;"] = nil - bh.sql2result["commit;"] = nil - bh.sql2result["rollback;"] = nil - - sql, _ := getSqlForCheckStageStatusWithStageName(ctx, "stage1") - mrs := newMrsForPasswordOfUser([][]interface{}{ - {"/tmp", "disabled"}, - }) - bh.sql2result[sql] = mrs - - err := doCheckFilePath(ctx, ses, cs.Ep) - convey.So(err, convey.ShouldNotBeNil) - }) - - convey.Convey("doCheckFilePath success", t, func() { - ctrl := gomock.NewController(t) - defer ctrl.Finish() - - ses := newTestSession(t, ctrl) - defer ses.Close() - - bh := &backgroundExecTest{} - bh.init() - - bhStub := gostub.StubFunc(&NewBackgroundExec, bh) - defer bhStub.Reset() - - pu := config.NewParameterUnit(&config.FrontendParameters{}, nil, nil, nil) - pu.SV.SetDefaultValues() - ctx := context.WithValue(context.TODO(), config.ParameterUnitKey, pu) - - rm, _ := NewRoutineManager(ctx) - ses.rm = rm - - tenant := &TenantInfo{ - Tenant: sysAccountName, - User: rootName, - DefaultRole: moAdminRoleName, - TenantID: sysAccountID, - UserID: rootID, - DefaultRoleID: moAdminRoleID, - } - ses.SetTenantInfo(tenant) - - cs := &tree.Select{ - Ep: &tree.ExportParam{ - FilePath: "stage1:/t1.csv", - }, - } - ses.InitExportConfig(cs.Ep) - - //no result set - bh.sql2result["begin;"] = nil - bh.sql2result["commit;"] = nil - bh.sql2result["rollback;"] = nil - - sql, _ := getSqlForCheckStageStatusWithStageName(ctx, "stage1") - mrs := newMrsForPasswordOfUser([][]interface{}{ - {"/tmp", "enabled"}, - }) - bh.sql2result[sql] = mrs - - err := doCheckFilePath(ctx, ses, cs.Ep) - convey.So(err, convey.ShouldBeNil) - convey.So(cs.Ep.FilePath, convey.ShouldEqual, "stage1:/t1.csv") - }) -} - func TestGetLabelPart(t *testing.T) { user1 := "user1" require.Equal(t, "", getLabelPart(user1)) diff --git a/pkg/frontend/export.go b/pkg/frontend/export.go index baf8bbadee6ea..9f30007674fbd 100644 --- a/pkg/frontend/export.go +++ b/pkg/frontend/export.go @@ -15,12 +15,10 @@ package frontend import ( - "bufio" "bytes" "context" "fmt" "io" - "os" "slices" "strconv" "strings" @@ -44,10 +42,7 @@ import ( type ExportConfig struct { // configs from user input userConfig *tree.ExportParam - // file handler - File *os.File - // bufio.writer - Writer *bufio.Writer + // curFileSize CurFileSize uint64 Rows uint64 @@ -56,26 +51,20 @@ type ExportConfig struct { Symbol [][]byte // default flush size DefaultBufSize int64 - OutputStr []byte - LineSize uint64 //file service & buffer for the line - UseFileService bool writeParam FileService fileservice.FileService - LineBuffer *bytes.Buffer Ctx context.Context AsyncReader *io.PipeReader AsyncWriter *io.PipeWriter AsyncGroup *errgroup.Group mrs *MysqlResultSet - lineStr []byte ctx context.Context } type writeParam struct { First bool - OutTofile bool Index atomic.Int32 WriteIndex atomic.Int32 ByteChan chan *BatchByte @@ -88,7 +77,6 @@ type BatchByte struct { err error } -var OpenFile = os.OpenFile var escape byte = '"' type CloseExportData struct { @@ -141,54 +129,61 @@ func initExportFileParam(ep *ExportConfig, mrs *MysqlResultSet) { } var openNewFile = func(ctx context.Context, ep *ExportConfig, mrs *MysqlResultSet) error { - lineSize := ep.LineSize var err error + var filePath string ep.CurFileSize = 0 - if !ep.UseFileService { - var filePath string - if len(ep.userConfig.StageFilePath) != 0 { - filePath = getExportFilePath(ep.userConfig.StageFilePath, ep.FileCnt) - } else { - filePath = getExportFilePath(ep.userConfig.FilePath, ep.FileCnt) - } - ep.File, err = OpenFile(filePath, os.O_RDWR|os.O_EXCL|os.O_CREATE, 0o666) + + ep.AsyncReader, ep.AsyncWriter = io.Pipe() + if len(ep.userConfig.StageFilePath) != 0 { + filePath = getExportFilePath(ep.userConfig.StageFilePath, ep.FileCnt) + } else { + filePath = getExportFilePath(ep.userConfig.FilePath, ep.FileCnt) + } + + // fileservice is determined from filepath + // if filepath is SHARED:/path, return getGlobalPu().FileService, filepath + // otherwise, return fileservice.GetForETL(ctx, nil, filepath) + fspath, err := fileservice.ParsePath(filePath) + if err != nil { + return err + } + + var readPath string + if fspath.Service == defines.SharedFileServiceName { + ep.FileService = getGlobalPu().FileService + readPath = filePath + + } else { + ep.FileService, readPath, err = fileservice.GetForETL(ctx, nil, filePath) if err != nil { return err } - ep.Writer = bufio.NewWriterSize(ep.File, int(ep.DefaultBufSize)) - } else { - //default 1MB - if ep.LineBuffer == nil { - ep.LineBuffer = &bytes.Buffer{} - } else { - ep.LineBuffer.Reset() - } - ep.AsyncReader, ep.AsyncWriter = io.Pipe() - filePath := getExportFilePath(ep.userConfig.FilePath, ep.FileCnt) - - asyncWriteFunc := func() error { - vec := fileservice.IOVector{ - FilePath: filePath, - Entries: []fileservice.IOEntry{ - { - ReaderForWrite: ep.AsyncReader, - Size: -1, - }, + + } + + asyncWriteFunc := func() error { + vec := fileservice.IOVector{ + FilePath: readPath, + Entries: []fileservice.IOEntry{ + { + ReaderForWrite: ep.AsyncReader, + Size: -1, }, + }, + } + err := ep.FileService.Write(ctx, vec) + if err != nil { + err2 := ep.AsyncReader.CloseWithError(err) + if err2 != nil { + return err2 } - err := ep.FileService.Write(ctx, vec) - if err != nil { - err2 := ep.AsyncReader.CloseWithError(err) - if err2 != nil { - return err2 - } - } - return err } - - ep.AsyncGroup, _ = errgroup.WithContext(ctx) - ep.AsyncGroup.Go(asyncWriteFunc) + return err } + + ep.AsyncGroup, _ = errgroup.WithContext(ctx) + ep.AsyncGroup.Go(asyncWriteFunc) + if ep.userConfig.Header { var header string n := len(mrs.Columns) @@ -205,17 +200,9 @@ var openNewFile = func(ctx context.Context, ep *ExportConfig, mrs *MysqlResultSe if err := writeDataToCSVFile(ep, []byte(header)); err != nil { return err } - if _, err := EndOfLine(ep); err != nil { - return err - } - } - if lineSize != 0 { - ep.LineSize = 0 - ep.Rows = 0 - if err := writeDataToCSVFile(ep, ep.OutputStr); err != nil { - return err - } } + + ep.Rows = 0 return nil } @@ -227,129 +214,61 @@ func getExportFilePath(filename string, fileCnt uint) string { } } -var formatOutputString = func(oq *ExportConfig, tmp, symbol []byte, enclosed byte, flag bool) error { +var formatOutputString = func(oq *ExportConfig, tmp, symbol []byte, enclosed byte, flag bool, buffer *bytes.Buffer) error { var err error if flag && enclosed != 0 { - if err = writeToCSVFile(oq, []byte{enclosed}); err != nil { + if _, err = buffer.Write([]byte{enclosed}); err != nil { return err } } - if err = writeToCSVFile(oq, tmp); err != nil { + if _, err = buffer.Write(tmp); err != nil { return err } if flag && enclosed != 0 { - if err = writeToCSVFile(oq, []byte{enclosed}); err != nil { + if _, err = buffer.Write([]byte{enclosed}); err != nil { return err } } - if err = writeToCSVFile(oq, symbol); err != nil { + if _, err = buffer.Write(symbol); err != nil { return err } return nil } -var Flush = func(ep *ExportConfig) error { - if !ep.UseFileService { - return ep.Writer.Flush() - } - return nil -} - -var Seek = func(ep *ExportConfig) (int64, error) { - if !ep.UseFileService { - return ep.File.Seek(int64(ep.CurFileSize-ep.LineSize), io.SeekStart) - } - return 0, nil -} - -var Read = func(ep *ExportConfig) (int, error) { - if !ep.UseFileService { - ep.OutputStr = make([]byte, ep.LineSize) - return ep.File.Read(ep.OutputStr) - } else { - ep.OutputStr = make([]byte, ep.LineSize) - copy(ep.OutputStr, ep.LineBuffer.Bytes()) - ep.LineBuffer.Reset() - return int(ep.LineSize), nil +var Close = func(ep *ExportConfig) error { + ep.FileCnt++ + err := ep.AsyncWriter.Close() + if err != nil { + return err } -} - -var Truncate = func(ep *ExportConfig) error { - if !ep.UseFileService { - return ep.File.Truncate(int64(ep.CurFileSize - ep.LineSize)) - } else { - return nil + err = ep.AsyncGroup.Wait() + if err != nil { + return err } -} - -var Close = func(ep *ExportConfig) error { - if !ep.UseFileService { - ep.FileCnt++ - return ep.File.Close() - } else { - ep.FileCnt++ - err := ep.AsyncWriter.Close() - if err != nil { - return err - } - err = ep.AsyncGroup.Wait() - if err != nil { - return err - } - err = ep.AsyncReader.Close() - if err != nil { - return err - } - ep.AsyncReader = nil - ep.AsyncWriter = nil - ep.AsyncGroup = nil + err = ep.AsyncReader.Close() + if err != nil { return err } + ep.AsyncReader = nil + ep.AsyncWriter = nil + ep.AsyncGroup = nil + return err } var Write = func(ep *ExportConfig, output []byte) (int, error) { - if !ep.UseFileService { - return ep.Writer.Write(output) - } else { - return ep.LineBuffer.Write(output) - } -} - -var EndOfLine = func(ep *ExportConfig) (int, error) { - if ep.UseFileService { - n, err := ep.AsyncWriter.Write(ep.LineBuffer.Bytes()) - if err != nil { - err2 := ep.AsyncWriter.CloseWithError(err) - if err2 != nil { - return 0, err2 - } + n, err := ep.AsyncWriter.Write(output) + if err != nil { + err2 := ep.AsyncWriter.CloseWithError(err) + if err2 != nil { + return 0, err2 } - ep.LineBuffer.Reset() - return n, err } - return 0, nil + return n, err } +// wrtieToCSVFile function may create a new file. Make sure the output buffer contains the complete CSV row to keep the CSV parser happy. func writeToCSVFile(ep *ExportConfig, output []byte) error { if ep.userConfig.MaxFileSize != 0 && ep.CurFileSize+uint64(len(output)) > ep.userConfig.MaxFileSize { - if err := Flush(ep); err != nil { - return err - } - if ep.LineSize != 0 && ep.OutTofile { - if _, err := Seek(ep); err != nil { - return err - } - for { - if n, err := Read(ep); err != nil { - return err - } else if uint64(n) == ep.LineSize { - break - } - } - if err := Truncate(ep); err != nil { - return err - } - } if err := Close(ep); err != nil { return err } @@ -365,6 +284,7 @@ func writeToCSVFile(ep *ExportConfig, output []byte) error { } var writeDataToCSVFile = func(ep *ExportConfig, output []byte) error { + for { if n, err := Write(ep, output); err != nil { return err @@ -372,23 +292,10 @@ var writeDataToCSVFile = func(ep *ExportConfig, output []byte) error { break } } - ep.LineSize += uint64(len(output)) ep.CurFileSize += uint64(len(output)) return nil } -func appendBytes(writeByte, tmp, symbol []byte, enclosed byte, flag bool) []byte { - if flag && enclosed != 0 { - writeByte = append(writeByte, enclosed) - } - writeByte = append(writeByte, tmp...) - if flag && enclosed != 0 { - writeByte = append(writeByte, enclosed) - } - writeByte = append(writeByte, symbol...) - return writeByte -} - func formatJsonString(str string, flag bool, terminatedBy string) string { if len(str) < 2 { return "\"" + str + "\"" @@ -408,23 +315,25 @@ func constructByte(ctx context.Context, obj FeSession, bat *batch.Batch, index i closeby := ep.userConfig.Fields.EnclosedBy.Value terminated := ep.userConfig.Fields.Terminated.Value flag := ep.ColumnFlag - writeByte := make([]byte, 0) + + buffer := &bytes.Buffer{} + for i := 0; i < bat.RowCount(); i++ { for j, vec := range bat.Vecs { if vec.GetNulls().Contains(uint64(i)) { - writeByte = appendBytes(writeByte, []byte("\\N"), symbol[j], closeby, flag[j]) + formatOutputString(ep, []byte("\\N"), symbol[j], closeby, flag[j], buffer) continue } switch vec.GetType().Oid { //get col case types.T_json: val := types.DecodeJson(vec.GetBytesAt(i)) - writeByte = appendBytes(writeByte, []byte(formatJsonString(val.String(), flag[j], terminated)), symbol[j], closeby, flag[j]) + formatOutputString(ep, []byte(formatJsonString(val.String(), flag[j], terminated)), symbol[j], closeby, flag[j], buffer) case types.T_bool: val := vector.GetFixedAt[bool](vec, i) if val { - writeByte = appendBytes(writeByte, []byte("true"), symbol[j], closeby, flag[j]) + formatOutputString(ep, []byte("true"), symbol[j], closeby, flag[j], buffer) } else { - writeByte = appendBytes(writeByte, []byte("false"), symbol[j], closeby, flag[j]) + formatOutputString(ep, []byte("false"), symbol[j], closeby, flag[j], buffer) } case types.T_bit: val := vector.GetFixedAt[uint64](vec, i) @@ -432,92 +341,92 @@ func constructByte(ctx context.Context, obj FeSession, bat *batch.Batch, index i byteLength := (bitLength + 7) / 8 b := types.EncodeUint64(&val)[:byteLength] slices.Reverse(b) - writeByte = appendBytes(writeByte, b, symbol[j], closeby, flag[j]) + formatOutputString(ep, b, symbol[j], closeby, flag[j], buffer) case types.T_int8: val := vector.GetFixedAt[int8](vec, i) - writeByte = appendBytes(writeByte, []byte(strconv.FormatInt(int64(val), 10)), symbol[j], closeby, flag[j]) + formatOutputString(ep, []byte(strconv.FormatInt(int64(val), 10)), symbol[j], closeby, flag[j], buffer) case types.T_int16: val := vector.GetFixedAt[int16](vec, i) - writeByte = appendBytes(writeByte, []byte(strconv.FormatInt(int64(val), 10)), symbol[j], closeby, flag[j]) + formatOutputString(ep, []byte(strconv.FormatInt(int64(val), 10)), symbol[j], closeby, flag[j], buffer) case types.T_int32: val := vector.GetFixedAt[int32](vec, i) - writeByte = appendBytes(writeByte, []byte(strconv.FormatInt(int64(val), 10)), symbol[j], closeby, flag[j]) + formatOutputString(ep, []byte(strconv.FormatInt(int64(val), 10)), symbol[j], closeby, flag[j], buffer) case types.T_int64: val := vector.GetFixedAt[int64](vec, i) - writeByte = appendBytes(writeByte, []byte(strconv.FormatInt(int64(val), 10)), symbol[j], closeby, flag[j]) + formatOutputString(ep, []byte(strconv.FormatInt(int64(val), 10)), symbol[j], closeby, flag[j], buffer) case types.T_uint8: val := vector.GetFixedAt[uint8](vec, i) - writeByte = appendBytes(writeByte, []byte(strconv.FormatUint(uint64(val), 10)), symbol[j], closeby, flag[j]) + formatOutputString(ep, []byte(strconv.FormatUint(uint64(val), 10)), symbol[j], closeby, flag[j], buffer) case types.T_uint16: val := vector.GetFixedAt[uint16](vec, i) - writeByte = appendBytes(writeByte, []byte(strconv.FormatUint(uint64(val), 10)), symbol[j], closeby, flag[j]) + formatOutputString(ep, []byte(strconv.FormatUint(uint64(val), 10)), symbol[j], closeby, flag[j], buffer) case types.T_uint32: val := vector.GetFixedAt[uint32](vec, i) - writeByte = appendBytes(writeByte, []byte(strconv.FormatUint(uint64(val), 10)), symbol[j], closeby, flag[j]) + formatOutputString(ep, []byte(strconv.FormatUint(uint64(val), 10)), symbol[j], closeby, flag[j], buffer) case types.T_uint64: val := vector.GetFixedAt[uint64](vec, i) - writeByte = appendBytes(writeByte, []byte(strconv.FormatUint(uint64(val), 10)), symbol[j], closeby, flag[j]) + formatOutputString(ep, []byte(strconv.FormatUint(uint64(val), 10)), symbol[j], closeby, flag[j], buffer) case types.T_float32: val := vector.GetFixedAt[float32](vec, i) if vec.GetType().Scale < 0 || vec.GetType().Width == 0 { - writeByte = appendBytes(writeByte, []byte(strconv.FormatFloat(float64(val), 'f', -1, 32)), symbol[j], closeby, flag[j]) + formatOutputString(ep, []byte(strconv.FormatFloat(float64(val), 'f', -1, 32)), symbol[j], closeby, flag[j], buffer) } else { - writeByte = appendBytes(writeByte, []byte(strconv.FormatFloat(float64(val), 'f', int(vec.GetType().Scale), 64)), symbol[j], closeby, flag[j]) + formatOutputString(ep, []byte(strconv.FormatFloat(float64(val), 'f', int(vec.GetType().Scale), 64)), symbol[j], closeby, flag[j], buffer) } case types.T_float64: val := vector.GetFixedAt[float64](vec, i) if vec.GetType().Scale < 0 || vec.GetType().Width == 0 { - writeByte = appendBytes(writeByte, []byte(strconv.FormatFloat(float64(val), 'f', -1, 32)), symbol[j], closeby, flag[j]) + formatOutputString(ep, []byte(strconv.FormatFloat(float64(val), 'f', -1, 32)), symbol[j], closeby, flag[j], buffer) } else { - writeByte = appendBytes(writeByte, []byte(strconv.FormatFloat(float64(val), 'f', int(vec.GetType().Scale), 64)), symbol[j], closeby, flag[j]) + formatOutputString(ep, []byte(strconv.FormatFloat(float64(val), 'f', int(vec.GetType().Scale), 64)), symbol[j], closeby, flag[j], buffer) } case types.T_char, types.T_varchar, types.T_blob, types.T_text, types.T_binary, types.T_varbinary, types.T_datalink: value := addEscapeToString(vec.GetBytesAt(i)) - writeByte = appendBytes(writeByte, value, symbol[j], closeby, true) + formatOutputString(ep, value, symbol[j], closeby, true, buffer) case types.T_array_float32: arrStr := types.BytesToArrayToString[float32](vec.GetBytesAt(i)) value := addEscapeToString(util2.UnsafeStringToBytes(arrStr)) - writeByte = appendBytes(writeByte, value, symbol[j], closeby, true) + formatOutputString(ep, value, symbol[j], closeby, true, buffer) case types.T_array_float64: arrStr := types.BytesToArrayToString[float64](vec.GetBytesAt(i)) value := addEscapeToString(util2.UnsafeStringToBytes(arrStr)) - writeByte = appendBytes(writeByte, value, symbol[j], closeby, true) + formatOutputString(ep, value, symbol[j], closeby, true, buffer) case types.T_date: val := vector.GetFixedAt[types.Date](vec, i) - writeByte = appendBytes(writeByte, []byte(val.String()), symbol[j], closeby, flag[j]) + formatOutputString(ep, []byte(val.String()), symbol[j], closeby, flag[j], buffer) case types.T_datetime: scale := vec.GetType().Scale val := vector.GetFixedAt[types.Datetime](vec, i).String2(scale) - writeByte = appendBytes(writeByte, []byte(val), symbol[j], closeby, flag[j]) + formatOutputString(ep, []byte(val), symbol[j], closeby, flag[j], buffer) case types.T_time: scale := vec.GetType().Scale val := vector.GetFixedAt[types.Time](vec, i).String2(scale) - writeByte = appendBytes(writeByte, []byte(val), symbol[j], closeby, flag[j]) + formatOutputString(ep, []byte(val), symbol[j], closeby, flag[j], buffer) case types.T_timestamp: scale := vec.GetType().Scale timeZone := ses.GetTimeZone() val := vector.GetFixedAt[types.Timestamp](vec, i).String2(timeZone, scale) - writeByte = appendBytes(writeByte, []byte(val), symbol[j], closeby, flag[j]) + formatOutputString(ep, []byte(val), symbol[j], closeby, flag[j], buffer) case types.T_decimal64: scale := vec.GetType().Scale val := vector.GetFixedAt[types.Decimal64](vec, i).Format(scale) - writeByte = appendBytes(writeByte, []byte(val), symbol[j], closeby, flag[j]) + formatOutputString(ep, []byte(val), symbol[j], closeby, flag[j], buffer) case types.T_decimal128: scale := vec.GetType().Scale val := vector.GetFixedAt[types.Decimal128](vec, i).Format(scale) - writeByte = appendBytes(writeByte, []byte(val), symbol[j], closeby, flag[j]) + formatOutputString(ep, []byte(val), symbol[j], closeby, flag[j], buffer) case types.T_uuid: val := vector.GetFixedAt[types.Uuid](vec, i).String() - writeByte = appendBytes(writeByte, []byte(val), symbol[j], closeby, flag[j]) + formatOutputString(ep, []byte(val), symbol[j], closeby, flag[j], buffer) case types.T_Rowid: val := vector.GetFixedAt[types.Rowid](vec, i) - writeByte = appendBytes(writeByte, []byte(val.String()), symbol[j], closeby, flag[j]) + formatOutputString(ep, []byte(val.String()), symbol[j], closeby, flag[j], buffer) case types.T_Blockid: val := vector.GetFixedAt[types.Blockid](vec, i) - writeByte = appendBytes(writeByte, []byte(val.String()), symbol[j], closeby, flag[j]) + formatOutputString(ep, []byte(val.String()), symbol[j], closeby, flag[j], buffer) case types.T_enum: val := vector.GetFixedAt[types.Enum](vec, i).String() - writeByte = appendBytes(writeByte, []byte(val), symbol[j], closeby, flag[j]) + formatOutputString(ep, []byte(val), symbol[j], closeby, flag[j], buffer) default: ses.Error(ctx, "Failed to construct byte due to unsupported type", @@ -531,13 +440,20 @@ func constructByte(ctx context.Context, obj FeSession, bat *batch.Batch, index i } } + // copy data. byteBuffer.Bytes() is not able to pass to channel + reslen := buffer.Len() + result := make([]byte, reslen) + copy(result, buffer.Bytes()) + buffer = nil + ByteChan <- &BatchByte{ index: index, - writeByte: writeByte, + writeByte: result, err: nil, } - ses.writeCsvBytes.Add(int64(len(writeByte))) // statistic out traffic, CASE 2: select into + ses.writeCsvBytes.Add(int64(reslen)) // statistic out traffic, CASE 2: select into bat.Clean(ses.GetMemPool()) + } func addEscapeToString(s []byte) []byte { @@ -561,16 +477,15 @@ func addEscapeToString(s []byte) []byte { return ret } -func exportDataToCSVFile(oq *ExportConfig) error { - if !oq.OutTofile { - return exportDataToCSVFile2(oq) - } - oq.LineSize = 0 - +func exportDataFromResultSetToCSVFile(oq *ExportConfig) error { symbol := oq.Symbol closeby := oq.userConfig.Fields.EnclosedBy.Value flag := oq.ColumnFlag + + buffer := &bytes.Buffer{} + for i := uint64(0); i < oq.mrs.GetColumnCount(); i++ { + column, err := oq.mrs.GetColumn(oq.ctx, i) if err != nil { return err @@ -583,7 +498,7 @@ func exportDataToCSVFile(oq *ExportConfig) error { return err } else if isNil { //NULL is output as \N - if err = formatOutputString(oq, []byte{'\\', 'N'}, symbol[i], closeby, false); err != nil { + if err = formatOutputString(oq, []byte{'\\', 'N'}, symbol[i], closeby, false, buffer); err != nil { return err } continue @@ -595,7 +510,7 @@ func exportDataToCSVFile(oq *ExportConfig) error { if err != nil { return err } - if err = formatOutputString(oq, []byte(value), symbol[i], closeby, flag[i]); err != nil { + if err = formatOutputString(oq, []byte(value), symbol[i], closeby, flag[i], buffer); err != nil { return err } case defines.MYSQL_TYPE_BOOL: @@ -603,7 +518,7 @@ func exportDataToCSVFile(oq *ExportConfig) error { if err != nil { return err } - if err = formatOutputString(oq, []byte(value), symbol[i], closeby, flag[i]); err != nil { + if err = formatOutputString(oq, []byte(value), symbol[i], closeby, flag[i], buffer); err != nil { return err } case defines.MYSQL_TYPE_BIT: @@ -611,7 +526,7 @@ func exportDataToCSVFile(oq *ExportConfig) error { if err != nil { return err } - if err = formatOutputString(oq, []byte(value), symbol[i], closeby, flag[i]); err != nil { + if err = formatOutputString(oq, []byte(value), symbol[i], closeby, flag[i], buffer); err != nil { return err } case defines.MYSQL_TYPE_TINY, defines.MYSQL_TYPE_SHORT, defines.MYSQL_TYPE_INT24, defines.MYSQL_TYPE_LONG, defines.MYSQL_TYPE_YEAR: @@ -621,20 +536,20 @@ func exportDataToCSVFile(oq *ExportConfig) error { } if mysqlColumn.ColumnType() == defines.MYSQL_TYPE_YEAR { if value == 0 { - if err = formatOutputString(oq, []byte("0000"), symbol[i], closeby, flag[i]); err != nil { + if err = formatOutputString(oq, []byte("0000"), symbol[i], closeby, flag[i], buffer); err != nil { return err } } else { - oq.resetLineStr() - oq.lineStr = strconv.AppendInt(oq.lineStr, value, 10) - if err = formatOutputString(oq, oq.lineStr, symbol[i], closeby, flag[i]); err != nil { + var lineStr []byte + lineStr = strconv.AppendInt(lineStr, value, 10) + if err = formatOutputString(oq, lineStr, symbol[i], closeby, flag[i], buffer); err != nil { return err } } } else { - oq.resetLineStr() - oq.lineStr = strconv.AppendInt(oq.lineStr, value, 10) - if err = formatOutputString(oq, oq.lineStr, symbol[i], closeby, flag[i]); err != nil { + var lineStr []byte + lineStr = strconv.AppendInt(lineStr, value, 10) + if err = formatOutputString(oq, lineStr, symbol[i], closeby, flag[i], buffer); err != nil { return err } } @@ -643,8 +558,8 @@ func exportDataToCSVFile(oq *ExportConfig) error { if err != nil { return err } - oq.lineStr = []byte(fmt.Sprintf("%v", value)) - if err = formatOutputString(oq, oq.lineStr, symbol[i], closeby, flag[i]); err != nil { + lineStr := []byte(fmt.Sprintf("%v", value)) + if err = formatOutputString(oq, lineStr, symbol[i], closeby, flag[i], buffer); err != nil { return err } case defines.MYSQL_TYPE_LONGLONG: @@ -652,9 +567,9 @@ func exportDataToCSVFile(oq *ExportConfig) error { if value, err := oq.mrs.GetUint64(oq.ctx, 0, i); err != nil { return err } else { - oq.resetLineStr() - oq.lineStr = strconv.AppendUint(oq.lineStr, value, 10) - if err = formatOutputString(oq, oq.lineStr, symbol[i], closeby, flag[i]); err != nil { + var lineStr []byte + lineStr = strconv.AppendUint(lineStr, value, 10) + if err = formatOutputString(oq, lineStr, symbol[i], closeby, flag[i], buffer); err != nil { return err } } @@ -662,9 +577,9 @@ func exportDataToCSVFile(oq *ExportConfig) error { if value, err := oq.mrs.GetInt64(oq.ctx, 0, i); err != nil { return err } else { - oq.resetLineStr() - oq.lineStr = strconv.AppendInt(oq.lineStr, value, 10) - if err = formatOutputString(oq, oq.lineStr, symbol[i], closeby, flag[i]); err != nil { + var lineStr []byte + lineStr = strconv.AppendInt(lineStr, value, 10) + if err = formatOutputString(oq, lineStr, symbol[i], closeby, flag[i], buffer); err != nil { return err } } @@ -677,7 +592,7 @@ func exportDataToCSVFile(oq *ExportConfig) error { return err } value = addEscapeToString(value.([]byte)) - if err = formatOutputString(oq, value.([]byte), symbol[i], closeby, true); err != nil { + if err = formatOutputString(oq, value.([]byte), symbol[i], closeby, true, buffer); err != nil { return err } case defines.MYSQL_TYPE_DATE: @@ -685,7 +600,7 @@ func exportDataToCSVFile(oq *ExportConfig) error { if err != nil { return err } - if err = formatOutputString(oq, []byte(value.(types.Date).String()), symbol[i], closeby, flag[i]); err != nil { + if err = formatOutputString(oq, []byte(value.(types.Date).String()), symbol[i], closeby, flag[i], buffer); err != nil { return err } case defines.MYSQL_TYPE_TIME: @@ -693,7 +608,7 @@ func exportDataToCSVFile(oq *ExportConfig) error { if err != nil { return err } - if err = formatOutputString(oq, []byte(value.(types.Time).String()), symbol[i], closeby, flag[i]); err != nil { + if err = formatOutputString(oq, []byte(value.(types.Time).String()), symbol[i], closeby, flag[i], buffer); err != nil { return err } case defines.MYSQL_TYPE_DATETIME: @@ -701,7 +616,7 @@ func exportDataToCSVFile(oq *ExportConfig) error { if err != nil { return err } - if err = formatOutputString(oq, []byte(value.(string)), symbol[i], closeby, flag[i]); err != nil { + if err = formatOutputString(oq, []byte(value.(string)), symbol[i], closeby, flag[i], buffer); err != nil { return err } case defines.MYSQL_TYPE_TIMESTAMP: @@ -709,7 +624,7 @@ func exportDataToCSVFile(oq *ExportConfig) error { if err != nil { return err } - if err = formatOutputString(oq, []byte(value), symbol[i], closeby, flag[i]); err != nil { + if err = formatOutputString(oq, []byte(value), symbol[i], closeby, flag[i], buffer); err != nil { return err } case defines.MYSQL_TYPE_JSON: @@ -718,7 +633,7 @@ func exportDataToCSVFile(oq *ExportConfig) error { return err } jsonStr := value.(bytejson.ByteJson).String() - if err = formatOutputString(oq, []byte(jsonStr), symbol[i], closeby, flag[i]); err != nil { + if err = formatOutputString(oq, []byte(jsonStr), symbol[i], closeby, flag[i], buffer); err != nil { return err } case defines.MYSQL_TYPE_UUID: @@ -726,7 +641,7 @@ func exportDataToCSVFile(oq *ExportConfig) error { if err != nil { return err } - if err = formatOutputString(oq, []byte(value), symbol[i], closeby, flag[i]); err != nil { + if err = formatOutputString(oq, []byte(value), symbol[i], closeby, flag[i], buffer); err != nil { return err } case defines.MYSQL_TYPE_ENUM: @@ -734,24 +649,30 @@ func exportDataToCSVFile(oq *ExportConfig) error { if err != nil { return err } - if err = formatOutputString(oq, []byte(value), symbol[i], closeby, flag[i]); err != nil { + if err = formatOutputString(oq, []byte(value), symbol[i], closeby, flag[i], buffer); err != nil { return err } default: return moerr.NewInternalError(oq.ctx, "unsupported column type %d ", mysqlColumn.ColumnType()) } } + + // write the complete CSV row to CSV Writer + err := writeToCSVFile(oq, buffer.Bytes()) + if err != nil { + return err + } oq.Rows++ - _, err := EndOfLine(oq) return err } -func exportDataToCSVFile2(ep *ExportConfig) error { +func exportDataFromBatchToCSVFile(ep *ExportConfig) error { var tmp *BatchByte select { case tmp = <-ep.ByteChan: default: } + if tmp != nil { if tmp.err != nil { return tmp.err @@ -764,16 +685,16 @@ func exportDataToCSVFile2(ep *ExportConfig) error { return nil } - if err := writeToCSVFile(ep, value); err != nil { + err := writeToCSVFile(ep, value) + if err != nil { return err } ep.WriteIndex.Add(1) ep.BatchMap[ep.WriteIndex.Load()] = nil - _, err := EndOfLine(ep) return err } -func exportAllData(ep *ExportConfig) error { +func exportAllDataFromBatches(ep *ExportConfig) error { var tmp *BatchByte for { tmp = nil @@ -800,6 +721,7 @@ func exportAllData(ep *ExportConfig) error { } ep.WriteIndex.Add(1) ep.BatchMap[ep.WriteIndex.Load()] = nil + } ep.First = false ep.FileCnt = 0 @@ -817,10 +739,6 @@ func (ec *ExportConfig) init() { ec.WriteIndex.Store(0) } -func (ec *ExportConfig) resetLineStr() { - ec.lineStr = ec.lineStr[:0] -} - func (ec *ExportConfig) Write(execCtx *ExecCtx, bat *batch.Batch) error { ec.Index.Add(1) copied, err := bat.Dup(execCtx.ses.GetMemPool()) @@ -828,7 +746,8 @@ func (ec *ExportConfig) Write(execCtx *ExecCtx, bat *batch.Batch) error { return err } go constructByte(execCtx.reqCtx, execCtx.ses, copied, ec.Index.Load(), ec.ByteChan, ec) - if err = exportDataToCSVFile(ec); err != nil { + + if err = exportDataFromBatchToCSVFile(ec); err != nil { execCtx.ses.Error(execCtx.reqCtx, "Error occurred while exporting to CSV file", zap.Error(err)) @@ -840,7 +759,6 @@ func (ec *ExportConfig) Write(execCtx *ExecCtx, bat *batch.Batch) error { func (ec *ExportConfig) Close() { if ec != nil { ec.mrs = nil - ec.lineStr = nil ec.ctx = nil } } diff --git a/pkg/frontend/export_test.go b/pkg/frontend/export_test.go index 6c67b75e6a8fe..53faf946d53b6 100644 --- a/pkg/frontend/export_test.go +++ b/pkg/frontend/export_test.go @@ -15,9 +15,8 @@ package frontend import ( - "bufio" + "bytes" "context" - "os" "testing" "github.com/prashantv/gostub" @@ -60,28 +59,6 @@ func Test_initExportFileParam(t *testing.T) { } func Test_openNewFile(t *testing.T) { - convey.Convey("openNewFile failed", t, func() { - ep := &ExportConfig{ - userConfig: &tree.ExportParam{ - Lines: &tree.Lines{ - TerminatedBy: &tree.Terminated{}, - }, - Fields: &tree.Fields{ - Terminated: &tree.Terminated{}, - EnclosedBy: &tree.EnclosedBy{}, - EscapedBy: &tree.EscapedBy{}, - }, - Header: true, - FilePath: "test/export.csv", - }, - mrs: &MysqlResultSet{}, - } - - stubs := gostub.StubFunc(&OpenFile, nil, moerr.NewInternalError(context.TODO(), "can not open file")) - defer stubs.Reset() - convey.So(openNewFile(context.TODO(), ep, ep.mrs), convey.ShouldNotBeNil) - }) - convey.Convey("openNewFile succ", t, func() { ep := &ExportConfig{ userConfig: &tree.ExportParam{ @@ -96,8 +73,7 @@ func Test_openNewFile(t *testing.T) { Header: true, FilePath: "test/export.csv", }, - LineSize: 1, - mrs: &MysqlResultSet{}, + mrs: &MysqlResultSet{}, } col1 := new(MysqlColumn) @@ -107,11 +83,7 @@ func Test_openNewFile(t *testing.T) { ep.mrs.AddColumn(col1) ep.mrs.AddColumn(col2) - var file = &os.File{} - stubs := gostub.StubFunc(&OpenFile, file, nil) - defer stubs.Reset() - - stubs = gostub.StubFunc(&writeDataToCSVFile, nil) + stubs := gostub.StubFunc(&writeDataToCSVFile, nil) defer stubs.Reset() convey.So(openNewFile(context.TODO(), ep, ep.mrs), convey.ShouldBeNil) @@ -133,16 +105,10 @@ func Test_formatOutputString(t *testing.T) { Header: true, FilePath: "test/export.csv", }, - LineSize: 1, - mrs: &MysqlResultSet{}, + mrs: &MysqlResultSet{}, } - stubs := gostub.StubFunc(&writeDataToCSVFile, moerr.NewInternalError(context.TODO(), "write err")) - defer stubs.Reset() - convey.So(formatOutputString(ep, nil, nil, '\n', true), convey.ShouldNotBeNil) - - stubs = gostub.StubFunc(&writeDataToCSVFile, nil) - defer stubs.Reset() - convey.So(formatOutputString(ep, nil, nil, '\n', true), convey.ShouldBeNil) + buffer := &bytes.Buffer{} + convey.So(formatOutputString(ep, nil, nil, '\n', true, buffer), convey.ShouldBeNil) }) } @@ -161,63 +127,34 @@ func Test_writeToCSVFile(t *testing.T) { Header: true, FilePath: "test/export.csv", }, - LineSize: 1, - Writer: &bufio.Writer{}, - mrs: &MysqlResultSet{}, + mrs: &MysqlResultSet{}, } var output = []byte{'1', '2'} ep.userConfig.MaxFileSize = 1 - convey.So(writeToCSVFile(ep, output), convey.ShouldNotBeNil) - - ep.Rows = 1 - stubs := gostub.StubFunc(&Flush, moerr.NewInternalError(context.TODO(), "Flush error")) - defer stubs.Reset() - - convey.So(writeToCSVFile(ep, output), convey.ShouldNotBeNil) - - stubs = gostub.StubFunc(&Flush, nil) - defer stubs.Reset() - - stubs = gostub.StubFunc(&Seek, int64(0), moerr.NewInternalError(context.TODO(), "Seek error")) - defer stubs.Reset() - convey.So(writeToCSVFile(ep, output), convey.ShouldNotBeNil) - - stubs = gostub.StubFunc(&Seek, int64(0), nil) - defer stubs.Reset() - stubs = gostub.StubFunc(&Read, 0, moerr.NewInternalError(context.TODO(), "Read error")) - defer stubs.Reset() - convey.So(writeToCSVFile(ep, output), convey.ShouldNotBeNil) - - stubs = gostub.StubFunc(&Read, 1, nil) - defer stubs.Reset() - - stubs = gostub.StubFunc(&Truncate, moerr.NewInternalError(context.TODO(), "Truncate error")) - defer stubs.Reset() - convey.So(writeToCSVFile(ep, output), convey.ShouldNotBeNil) - - stubs = gostub.StubFunc(&Truncate, nil) - defer stubs.Reset() - stubs = gostub.StubFunc(&Close, moerr.NewInternalError(context.TODO(), "Close error")) + stubs := gostub.StubFunc(&Close, moerr.NewInternalError(context.TODO(), "Close error")) defer stubs.Reset() convey.So(writeToCSVFile(ep, output), convey.ShouldNotBeNil) + // set Close to nil and openNewFile to err stubs = gostub.StubFunc(&Close, nil) defer stubs.Reset() stubs = gostub.StubFunc(&openNewFile, moerr.NewInternalError(context.TODO(), "openNewFile error")) defer stubs.Reset() convey.So(writeToCSVFile(ep, output), convey.ShouldNotBeNil) - stubs = gostub.StubFunc(&openNewFile, nil) + // set Close() to nil + stubs = gostub.StubFunc(&Close, nil) defer stubs.Reset() - stubs = gostub.StubFunc(&writeDataToCSVFile, moerr.NewInternalError(context.TODO(), "writeDataToCSVFile error")) + // set openNewFile() to nil + stubs = gostub.StubFunc(&openNewFile, nil) defer stubs.Reset() - convey.So(writeToCSVFile(ep, output), convey.ShouldNotBeNil) - + // set writeDataToCSVFile to nil stubs = gostub.StubFunc(&writeDataToCSVFile, nil) defer stubs.Reset() convey.So(writeToCSVFile(ep, output), convey.ShouldBeNil) + }) } @@ -236,9 +173,7 @@ func Test_writeDataToCSVFile(t *testing.T) { Header: true, FilePath: "test/export.csv", }, - LineSize: 1, - Writer: &bufio.Writer{}, - mrs: &MysqlResultSet{}, + mrs: &MysqlResultSet{}, } var output = []byte{'1', '2'} @@ -255,7 +190,7 @@ func Test_writeDataToCSVFile(t *testing.T) { } func Test_exportDataToCSVFile(t *testing.T) { - convey.Convey("exportDataToCSVFile succ", t, func() { + convey.Convey("exportDataFromResultSetToCSVFile succ", t, func() { ep := &ExportConfig{ userConfig: &tree.ExportParam{ Lines: &tree.Lines{ @@ -269,9 +204,7 @@ func Test_exportDataToCSVFile(t *testing.T) { Header: true, FilePath: "test/export.csv", }, - LineSize: 1, - Writer: &bufio.Writer{}, - mrs: &MysqlResultSet{}, + mrs: &MysqlResultSet{}, } var col = make([]MysqlColumn, 13) @@ -303,10 +236,14 @@ func Test_exportDataToCSVFile(t *testing.T) { ep.Symbol = make([][]byte, len(col)) ep.ColumnFlag = make([]bool, len(col)) - stubs := gostub.StubFunc(&formatOutputString, nil) + stubs := gostub.StubFunc(&Close, nil) + defer stubs.Reset() + stubs = gostub.StubFunc(&openNewFile, nil) + defer stubs.Reset() + stubs = gostub.StubFunc(&writeDataToCSVFile, nil) defer stubs.Reset() - convey.So(exportDataToCSVFile(ep), convey.ShouldBeNil) + convey.So(exportDataFromResultSetToCSVFile(ep), convey.ShouldBeNil) }) convey.Convey("exportDataToCSVFile fail", t, func() { @@ -323,9 +260,7 @@ func Test_exportDataToCSVFile(t *testing.T) { Header: true, FilePath: "test/export.csv", }, - LineSize: 1, - Writer: &bufio.Writer{}, - mrs: &MysqlResultSet{}, + mrs: &MysqlResultSet{}, } var col = make([]MysqlColumn, 1) @@ -341,9 +276,12 @@ func Test_exportDataToCSVFile(t *testing.T) { ep.Symbol = make([][]byte, len(col)) ep.ColumnFlag = make([]bool, len(col)) - stubs := gostub.StubFunc(&formatOutputString, nil) + stubs := gostub.StubFunc(&Close, nil) defer stubs.Reset() - - convey.So(exportDataToCSVFile(ep), convey.ShouldBeNil) + stubs = gostub.StubFunc(&openNewFile, nil) + defer stubs.Reset() + stubs = gostub.StubFunc(&writeDataToCSVFile, nil) + defer stubs.Reset() + convey.So(exportDataFromResultSetToCSVFile(ep), convey.ShouldBeNil) }) } diff --git a/pkg/frontend/query_result.go b/pkg/frontend/query_result.go index a2f7605999bd2..c2b47c34bfcce 100644 --- a/pkg/frontend/query_result.go +++ b/pkg/frontend/query_result.go @@ -597,15 +597,11 @@ func doDumpQueryResult(ctx context.Context, ses *Session, eParam *tree.ExportPar mrs: mrs, } //prepare output queue - exportParam.OutTofile = true //prepare export param exportParam.DefaultBufSize = getGlobalPu().SV.ExportDataDefaultFlushSize - exportParam.UseFileService = true exportParam.FileService = getGlobalPu().FileService exportParam.Ctx = ctx defer func() { - exportParam.LineBuffer = nil - exportParam.OutputStr = nil if exportParam.AsyncReader != nil { _ = exportParam.AsyncReader.Close() } @@ -665,7 +661,7 @@ func doDumpQueryResult(ctx context.Context, ses *Session, eParam *tree.ExportPar if err != nil { return err } - err = exportDataToCSVFile(exportParam) + err = exportDataFromResultSetToCSVFile(exportParam) if err != nil { return err } diff --git a/pkg/frontend/session_test.go b/pkg/frontend/session_test.go index 425a05698015e..d03a5adceafaa 100644 --- a/pkg/frontend/session_test.go +++ b/pkg/frontend/session_test.go @@ -492,11 +492,13 @@ func TestSession_updateTimeZone(t *testing.T) { ses := newSes(nil, ctrl) ctx := context.Background() - err := updateTimeZone(ctx, ses, ses.GetSessionSysVars(), "time_zone", "system") - assert.NoError(t, err) - assert.Equal(t, ses.GetTimeZone().String(), "Local") + /* + err := updateTimeZone(ctx, ses, ses.GetSessionSysVars(), "time_zone", "system") + assert.NoError(t, err) + assert.Equal(t, "Local", ses.GetTimeZone().String()) + */ - err = updateTimeZone(ctx, ses, ses.GetSessionSysVars(), "time_zone", "+00:00") + err := updateTimeZone(ctx, ses, ses.GetSessionSysVars(), "time_zone", "+00:00") assert.NoError(t, err) assert.Equal(t, ses.GetTimeZone().String(), "FixedZone") diff --git a/pkg/frontend/status_stmt.go b/pkg/frontend/status_stmt.go index 017e5d8c610b5..a613785f61966 100644 --- a/pkg/frontend/status_stmt.go +++ b/pkg/frontend/status_stmt.go @@ -73,13 +73,11 @@ func executeStatusStmt(ses *Session, execCtx *ExecCtx) (err error) { ses.Infof(execCtx.reqCtx, "time of Exec.Run : %s", time.Since(runBegin).String()) } - if err = exportAllData(ep); err != nil { + if err = exportAllDataFromBatches(ep); err != nil { return } - if err = ep.Writer.Flush(); err != nil { - return - } - if err = ep.File.Close(); err != nil { + + if err = Close(ep); err != nil { return } diff --git a/pkg/sql/colexec/table_function/stage.go b/pkg/sql/colexec/table_function/stage.go new file mode 100644 index 0000000000000..02db432cebcf9 --- /dev/null +++ b/pkg/sql/colexec/table_function/stage.go @@ -0,0 +1,137 @@ +// Copyright 2022 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package table_function + +import ( + "fmt" + "path" + "strings" + + "github.com/matrixorigin/matrixone/pkg/common/moerr" + "github.com/matrixorigin/matrixone/pkg/container/batch" + "github.com/matrixorigin/matrixone/pkg/container/types" + "github.com/matrixorigin/matrixone/pkg/container/vector" + "github.com/matrixorigin/matrixone/pkg/fileservice" + "github.com/matrixorigin/matrixone/pkg/sql/colexec" + "github.com/matrixorigin/matrixone/pkg/sql/plan/function" + "github.com/matrixorigin/matrixone/pkg/vm" + "github.com/matrixorigin/matrixone/pkg/vm/process" +) + +// prepare +func stageListPrepare(proc *process.Process, tableFunction *TableFunction) (err error) { + tableFunction.ctr.executorsForArgs, err = colexec.NewExpressionExecutorsFromPlanExpressions(proc, tableFunction.Args) + + for i := range tableFunction.Attrs { + tableFunction.Attrs[i] = strings.ToUpper(tableFunction.Attrs[i]) + } + return err +} + +// call +func stageListCall(_ int, proc *process.Process, tableFunction *TableFunction, result *vm.CallResult) (bool, error) { + + var ( + err error + rbat *batch.Batch + ) + bat := result.Batch + defer func() { + if err != nil && rbat != nil { + rbat.Clean(proc.Mp()) + } + }() + if bat == nil { + return true, nil + } + + v, err := tableFunction.ctr.executorsForArgs[0].Eval(proc, []*batch.Batch{bat}, nil) + if err != nil { + return false, err + } + if v.GetType().Oid != types.T_varchar { + return false, moerr.NewInvalidInput(proc.Ctx, fmt.Sprintf("stage_list: filepath must be string, but got %s", v.GetType().String())) + } + + filepath := v.UnsafeGetStringAt(0) + + rbat, err = stageList(proc, tableFunction, filepath) + if err != nil { + return false, err + } + + result.Batch = rbat + return false, nil +} + +func stageList(proc *process.Process, tableFunction *TableFunction, filepath string) (bat *batch.Batch, err error) { + + if len(tableFunction.Attrs) != 1 { + return nil, moerr.NewInvalidInput(proc.Ctx, "stage_list: number of output column must be 1.") + } + + bat = batch.NewWithSize(len(tableFunction.Attrs)) + bat.Attrs = tableFunction.Attrs + bat.Cnt = 1 + for i := range tableFunction.ctr.retSchema { + bat.Vecs[i] = proc.GetVector(tableFunction.ctr.retSchema[i]) + } + + rs := bat.GetVector(0) + + if len(filepath) == 0 { + if err := vector.AppendBytes(rs, nil, true, proc.Mp()); err != nil { + return nil, err + } + return bat, nil + } + + s, err := function.UrlToStageDef(string(filepath), proc) + if err != nil { + return nil, err + } + + fspath, _, err := s.ToPath() + if err != nil { + return nil, err + } + + idx := strings.LastIndex(fspath, fileservice.ServiceNameSeparator) + + var service, pattern string + if idx == -1 { + service = "" + pattern = fspath + } else { + service = fspath[:idx] + pattern = fspath[idx+1:] + } + + pattern = path.Clean("/" + pattern) + + fileList, err := function.StageListWithPattern(service, pattern, proc) + if err != nil { + return nil, err + } + + for _, f := range fileList { + if err := vector.AppendBytes(rs, []byte(f), false, proc.Mp()); err != nil { + return nil, err + } + } + + bat.SetRowCount(len(fileList)) + return bat, nil +} diff --git a/pkg/sql/colexec/table_function/table_function.go b/pkg/sql/colexec/table_function/table_function.go index c360a77efe748..3cd24c697f8a3 100644 --- a/pkg/sql/colexec/table_function/table_function.go +++ b/pkg/sql/colexec/table_function/table_function.go @@ -71,6 +71,8 @@ func (tableFunction *TableFunction) Call(proc *process.Process) (vm.CallResult, f, e = moTransactionsCall(idx, proc, tblArg, &result) case "mo_cache": f, e = moCacheCall(idx, proc, tblArg, &result) + case "stage_list": + f, e = stageListCall(idx, proc, tblArg, &result) default: result.Status = vm.ExecStop return result, moerr.NewNotSupported(proc.Ctx, fmt.Sprintf("table function %s is not supported", tblArg.FuncName)) @@ -154,6 +156,8 @@ func (tableFunction *TableFunction) Prepare(proc *process.Process) error { return moTransactionsPrepare(proc, tblArg) case "mo_cache": return moCachePrepare(proc, tblArg) + case "stage_list": + return stageListPrepare(proc, tblArg) default: return moerr.NewNotSupported(proc.Ctx, fmt.Sprintf("table function %s is not supported", tblArg.FuncName)) } diff --git a/pkg/sql/compile/compile.go b/pkg/sql/compile/compile.go index ffa4ca1337ead..0a7ac770e9a25 100644 --- a/pkg/sql/compile/compile.go +++ b/pkg/sql/compile/compile.go @@ -1499,9 +1499,24 @@ func (c *Compile) compileExternScan(n *plan.Node) ([]*Scope, error) { } else if param.ScanType == tree.INLINE { return c.compileExternValueScan(n, param, strictSqlMode) } else { - if err := plan2.InitInfileParam(param); err != nil { + if err := plan2.InitInfileOrStageParam(param, c.proc); err != nil { return nil, err } + + // if filepath is stage URL, ScanType may change to tree.S3. check param.Parallel again + if param.ScanType == tree.S3 && param.Parallel { + mcpu = 0 + ID2Addr = make(map[int]int, 0) + for i := 0; i < len(c.cnList); i++ { + tmp := mcpu + if c.cnList[i].Mcpu > external.S3ParallelMaxnum { + mcpu += external.S3ParallelMaxnum + } else { + mcpu += c.cnList[i].Mcpu + } + ID2Addr[i] = mcpu - tmp + } + } } t = time.Now() diff --git a/pkg/sql/plan/build_load.go b/pkg/sql/plan/build_load.go index 1cd24e2b41e4e..ae4fd9612e4b6 100644 --- a/pkg/sql/plan/build_load.go +++ b/pkg/sql/plan/build_load.go @@ -289,7 +289,7 @@ func checkFileExist(param *tree.ExternParam, ctx CompilerContext) (string, error return "", err } } else { - if err := InitInfileParam(param); err != nil { + if err := InitInfileOrStageParam(param, ctx.GetProcess()); err != nil { return "", err } } diff --git a/pkg/sql/plan/external.go b/pkg/sql/plan/external.go index e89ca57897120..43f003ba198ae 100644 --- a/pkg/sql/plan/external.go +++ b/pkg/sql/plan/external.go @@ -185,7 +185,7 @@ func getExternalStats(node *plan.Node, builder *QueryBuilder) *Stats { return DefaultHugeStats() } } else { - if err = InitInfileParam(param); err != nil { + if err = InitInfileOrStageParam(param, builder.compCtx.GetProcess()); err != nil { return DefaultHugeStats() } } diff --git a/pkg/sql/plan/function/stage_util.go b/pkg/sql/plan/function/stage_util.go new file mode 100644 index 0000000000000..f9c2b6e52c316 --- /dev/null +++ b/pkg/sql/plan/function/stage_util.go @@ -0,0 +1,430 @@ +// Copyright 2021 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package function + +import ( + "container/list" + "context" + "encoding/csv" + "fmt" + "net/url" + "path" + "strings" + + "github.com/matrixorigin/matrixone/pkg/common/moerr" + moruntime "github.com/matrixorigin/matrixone/pkg/common/runtime" + "github.com/matrixorigin/matrixone/pkg/container/vector" + "github.com/matrixorigin/matrixone/pkg/fileservice" + //"github.com/matrixorigin/matrixone/pkg/logutil" + "github.com/matrixorigin/matrixone/pkg/util/executor" + "github.com/matrixorigin/matrixone/pkg/vm/process" +) + +const STAGE_PROTOCOL = "stage" +const S3_PROTOCOL = "s3" +const FILE_PROTOCOL = "file" + +const PARAMKEY_AWS_KEY_ID = "aws_key_id" +const PARAMKEY_AWS_SECRET_KEY = "aws_secret_key" +const PARAMKEY_AWS_REGION = "aws_region" +const PARAMKEY_ENDPOINT = "endpoint" +const PARAMKEY_COMPRESSION = "compression" +const PARAMKEY_PROVIDER = "provider" + +const S3_PROVIDER_AMAZON = "amazon" +const S3_PROVIDER_MINIO = "minio" + +const S3_SERVICE = "s3" +const MINIO_SERVICE = "minio" + +type StageDef struct { + Id uint32 + Name string + Url *url.URL + Credentials map[string]string + Status string +} + +func (s *StageDef) GetCredentials(key string, defval string) (string, bool) { + if s.Credentials == nil { + // no credential in this stage + return defval, false + } + + k := strings.ToLower(key) + res, ok := s.Credentials[k] + if !ok { + return defval, false + } + return res, ok +} + +func (s *StageDef) expandSubStage(proc *process.Process) (StageDef, error) { + if s.Url.Scheme == STAGE_PROTOCOL { + stagename, prefix, query, err := ParseStageUrl(s.Url) + if err != nil { + return StageDef{}, err + } + + res, err := StageLoadCatalog(proc, stagename) + if err != nil { + return StageDef{}, err + } + + res.Url = res.Url.JoinPath(prefix) + res.Url.RawQuery = query + return res.expandSubStage(proc) + } + + return *s, nil +} + +// get stages and expand the path. stage may be a file or s3 +// use the format of path s3,,,,,, +// or minio,,,,,, +// expand the subpath to MO path. +// subpath is in the format like path or path with query like path?q1=v1&q2=v2... +func (s *StageDef) ToPath() (mopath string, query string, err error) { + + if s.Url.Scheme == S3_PROTOCOL { + bucket, prefix, query, err := ParseS3Url(s.Url) + if err != nil { + return "", "", err + } + + // get S3 credentials + aws_key_id, found := s.GetCredentials(PARAMKEY_AWS_KEY_ID, "") + if !found { + return "", "", moerr.NewBadConfig(context.TODO(), "Stage credentials: AWS_KEY_ID not found") + } + aws_secret_key, found := s.GetCredentials(PARAMKEY_AWS_SECRET_KEY, "") + if !found { + return "", "", moerr.NewBadConfig(context.TODO(), "Stage credentials: AWS_SECRET_KEY not found") + } + aws_region, found := s.GetCredentials(PARAMKEY_AWS_REGION, "") + if !found { + return "", "", moerr.NewBadConfig(context.TODO(), "Stage credentials: AWS_REGION not found") + } + provider, found := s.GetCredentials(PARAMKEY_PROVIDER, "") + if !found { + return "", "", moerr.NewBadConfig(context.TODO(), "Stage credentials: PROVIDER not found") + } + endpoint, found := s.GetCredentials(PARAMKEY_ENDPOINT, "") + if !found { + return "", "", moerr.NewBadConfig(context.TODO(), "Stage credentials: ENDPOINT not found") + } + + service, err := getS3ServiceFromProvider(provider) + if err != nil { + return "", "", err + } + + buf := new(strings.Builder) + w := csv.NewWriter(buf) + opts := []string{service, endpoint, aws_region, bucket, aws_key_id, aws_secret_key, ""} + + if err = w.Write(opts); err != nil { + return "", "", err + } + w.Flush() + return fileservice.JoinPath(buf.String(), prefix), query, nil + } else if s.Url.Scheme == FILE_PROTOCOL { + return s.Url.Path, s.Url.RawQuery, nil + } + return "", "", moerr.NewBadConfig(context.TODO(), "URL protocol %s not supported", s.Url.Scheme) +} + +func getS3ServiceFromProvider(provider string) (string, error) { + provider = strings.ToLower(provider) + switch provider { + case S3_PROVIDER_AMAZON: + return S3_SERVICE, nil + case S3_PROVIDER_MINIO: + return MINIO_SERVICE, nil + default: + return "", moerr.NewBadConfig(context.TODO(), "provider %s not supported", provider) + } +} + +func runSql(proc *process.Process, sql string) (executor.Result, error) { + v, ok := moruntime.ServiceRuntime(proc.GetService()).GetGlobalVariables(moruntime.InternalSQLExecutor) + if !ok { + panic("missing lock service") + } + exec := v.(executor.SQLExecutor) + opts := executor.Options{}. + // All runSql and runSqlWithResult is a part of input sql, can not incr statement. + // All these sub-sql's need to be rolled back and retried en masse when they conflict in pessimistic mode + WithDisableIncrStatement(). + WithTxn(proc.GetTxnOperator()). + WithDatabase(proc.GetSessionInfo().Database). + WithTimeZone(proc.GetSessionInfo().TimeZone). + WithAccountID(proc.GetSessionInfo().AccountId) + return exec.Exec(proc.GetTopContext(), sql, opts) +} + +func credentialsToMap(cred string) (map[string]string, error) { + if len(cred) == 0 { + return nil, nil + } + + opts := strings.Split(cred, ",") + if len(opts) == 0 { + return nil, nil + } + + credentials := make(map[string]string) + for _, o := range opts { + kv := strings.SplitN(o, "=", 2) + if len(kv) != 2 { + return nil, moerr.NewBadConfig(context.TODO(), "Format error: invalid stage credentials") + } + credentials[strings.ToLower(kv[0])] = kv[1] + } + + return credentials, nil +} + +func StageLoadCatalog(proc *process.Process, stagename string) (s StageDef, err error) { + getAllStagesSql := fmt.Sprintf("select stage_id, stage_name, url, stage_credentials, stage_status from `%s`.`%s` WHERE stage_name = '%s';", "mo_catalog", "mo_stages", stagename) + res, err := runSql(proc, getAllStagesSql) + if err != nil { + return StageDef{}, err + } + defer res.Close() + + var reslist []StageDef + const id_idx = 0 + const name_idx = 1 + const url_idx = 2 + const cred_idx = 3 + const status_idx = 4 + if res.Batches != nil { + for _, batch := range res.Batches { + if batch != nil && batch.Vecs[0] != nil && batch.Vecs[0].Length() > 0 { + for i := 0; i < batch.Vecs[0].Length(); i++ { + stage_id := vector.GetFixedAt[uint32](batch.Vecs[id_idx], i) + stage_name := string(batch.Vecs[name_idx].GetBytesAt(i)) + stage_url, err := url.Parse(string(batch.Vecs[url_idx].GetBytesAt(i))) + if err != nil { + return StageDef{}, err + } + stage_cred := string(batch.Vecs[cred_idx].GetBytesAt(i)) + + credmap, err := credentialsToMap(stage_cred) + if err != nil { + return StageDef{}, err + } + + stage_status := string(batch.Vecs[status_idx].GetBytesAt(i)) + + //logutil.Infof("CATALOG: ID %d, stage %s url %s cred %s", stage_id, stage_name, stage_url, stage_cred) + reslist = append(reslist, StageDef{stage_id, stage_name, stage_url, credmap, stage_status}) + } + } + } + } + + if reslist == nil { + return StageDef{}, moerr.NewBadConfig(context.TODO(), "Stage %s not found", stagename) + } + + return reslist[0], nil +} + +func UrlToPath(furl string, proc *process.Process) (path string, query string, err error) { + + s, err := UrlToStageDef(furl, proc) + if err != nil { + return "", "", err + } + + return s.ToPath() +} + +func ParseStageUrl(u *url.URL) (stagename, prefix, query string, err error) { + if u.Scheme != STAGE_PROTOCOL { + return "", "", "", moerr.NewBadConfig(context.TODO(), "ParseStageUrl: URL protocol is not stage://") + } + + stagename = u.Host + if len(stagename) == 0 { + return "", "", "", moerr.NewBadConfig(context.TODO(), "Invalid stage URL: stage name is empty string") + } + + prefix = u.Path + query = u.RawQuery + + return +} + +func ParseS3Url(u *url.URL) (bucket, fpath, query string, err error) { + bucket = u.Host + fpath = u.Path + query = u.RawQuery + err = nil + + if len(bucket) == 0 { + err = moerr.NewBadConfig(context.TODO(), "Invalid s3 URL: bucket is empty string") + return "", "", "", err + } + + return +} + +func UrlToStageDef(furl string, proc *process.Process) (s StageDef, err error) { + + aurl, err := url.Parse(furl) + if err != nil { + return StageDef{}, err + } + + if aurl.Scheme != STAGE_PROTOCOL { + return StageDef{}, moerr.NewBadConfig(context.TODO(), "URL is not stage URL") + } + + stagename, subpath, query, err := ParseStageUrl(aurl) + if err != nil { + return StageDef{}, err + } + + sdef, err := StageLoadCatalog(proc, stagename) + if err != nil { + return StageDef{}, err + } + + s, err = sdef.expandSubStage(proc) + if err != nil { + return StageDef{}, err + } + + s.Url = s.Url.JoinPath(subpath) + s.Url.RawQuery = query + + return s, nil +} + +func stageListWithWildcard(service string, pattern string, proc *process.Process) (fileList []string, err error) { + const wildcards = "*?" + const sep = "/" + fs := proc.GetFileService() + + idx := strings.IndexAny(pattern, wildcards) + if idx == -1 { + return nil, moerr.NewInternalError(proc.Ctx, "pattern without wildcard") + } + + var pathDir []string + idx = strings.LastIndex(pattern[:idx], sep) + if idx == -1 { + pathDir = append(pathDir, "") + pathDir = append(pathDir, strings.Split(pattern, sep)...) + } else { + pathDir = append(pathDir, pattern[:idx]) + pathDir = append(pathDir, strings.Split(pattern[idx+1:], sep)...) + } + + l := list.New() + l2 := list.New() + if pathDir[0] == "" { + l.PushBack(sep) + } else { + l.PushBack(pathDir[0]) + } + + for i := 1; i < len(pathDir); i++ { + length := l.Len() + for j := 0; j < length; j++ { + prefix := l.Front().Value.(string) + p := fileservice.JoinPath(service, prefix) + etlfs, readpath, err := fileservice.GetForETL(proc.Ctx, fs, p) + if err != nil { + return nil, err + } + entries, err := etlfs.List(proc.Ctx, readpath) + if err != nil { + return nil, err + } + for _, entry := range entries { + if !entry.IsDir && i+1 != len(pathDir) { + continue + } + if entry.IsDir && i+1 == len(pathDir) { + continue + } + matched, err := path.Match(pathDir[i], entry.Name) + if err != nil { + return nil, err + } + if !matched { + continue + } + l.PushBack(path.Join(l.Front().Value.(string), entry.Name)) + if !entry.IsDir { + l2.PushBack(entry.Size) + } + } + l.Remove(l.Front()) + } + } + length := l.Len() + + for j := 0; j < length; j++ { + fileList = append(fileList, l.Front().Value.(string)) + l.Remove(l.Front()) + //fileSize = append(fileSize, l2.Front().Value.(int64)) + l2.Remove(l2.Front()) + } + + return fileList, nil +} + +func stageListWithoutWildcard(service string, pattern string, proc *process.Process) (fileList []string, err error) { + + fs := proc.GetFileService() + p := fileservice.JoinPath(service, pattern) + etlfs, readpath, err := fileservice.GetForETL(proc.Ctx, fs, p) + if err != nil { + return nil, err + } + entries, err := etlfs.List(proc.Ctx, readpath) + if err != nil { + return nil, err + } + for _, entry := range entries { + fileList = append(fileList, path.Join(pattern, entry.Name)) + } + + return fileList, nil +} + +func StageListWithPattern(service string, pattern string, proc *process.Process) (fileList []string, err error) { + const wildcards = "*?" + + idx := strings.IndexAny(pattern, wildcards) + if idx == -1 { + // no wildcard in pattern + fileList, err = stageListWithoutWildcard(service, pattern, proc) + if err != nil { + return nil, err + } + } else { + fileList, err = stageListWithWildcard(service, pattern, proc) + if err != nil { + return nil, err + } + } + return fileList, nil +} diff --git a/pkg/sql/plan/query_builder.go b/pkg/sql/plan/query_builder.go index a233aa44570dc..c1692bc23ca3e 100644 --- a/pkg/sql/plan/query_builder.go +++ b/pkg/sql/plan/query_builder.go @@ -4099,6 +4099,8 @@ func (builder *QueryBuilder) buildTableFunction(tbl *tree.TableFunction, ctx *Bi nodeId, err = builder.buildMoTransactions(tbl, ctx, exprs, childId) case "mo_cache": nodeId, err = builder.buildMoCache(tbl, ctx, exprs, childId) + case "stage_list": + nodeId, err = builder.buildStageList(tbl, ctx, exprs, childId) default: err = moerr.NewNotSupported(builder.GetContext(), "table function '%s' not supported", id) } diff --git a/pkg/sql/plan/stage.go b/pkg/sql/plan/stage.go new file mode 100644 index 0000000000000..c251595293dee --- /dev/null +++ b/pkg/sql/plan/stage.go @@ -0,0 +1,54 @@ +// Copyright 2022 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package plan + +import ( + "github.com/matrixorigin/matrixone/pkg/container/types" + "github.com/matrixorigin/matrixone/pkg/pb/plan" + "github.com/matrixorigin/matrixone/pkg/sql/parsers/tree" +) + +var ( + stageListColDefs = []*plan.ColDef{ + { + Name: "file", + Typ: plan.Type{ + Id: int32(types.T_varchar), + NotNullable: false, + Width: types.MaxVarcharLen, + }, + }, + } +) + +func (builder *QueryBuilder) buildStageList(tbl *tree.TableFunction, ctx *BindContext, exprs []*plan.Expr, childId int32) (int32, error) { + colDefs := _getColDefs(stageListColDefs) + node := &plan.Node{ + NodeType: plan.Node_FUNCTION_SCAN, + Stats: &plan.Stats{}, + TableDef: &plan.TableDef{ + TableType: "func_table", //test if ok + //Name: tbl.String(), + TblFunc: &plan.TableFunction{ + Name: "stage_list", + }, + Cols: colDefs, + }, + BindingTags: []int32{builder.genNewTag()}, + TblFuncExprList: exprs, + Children: []int32{childId}, + } + return builder.appendNode(node, ctx), nil +} diff --git a/pkg/sql/plan/unnest.go b/pkg/sql/plan/unnest.go index 7c8b081c39071..4d25de1e9ba23 100644 --- a/pkg/sql/plan/unnest.go +++ b/pkg/sql/plan/unnest.go @@ -97,16 +97,16 @@ func _dupColDef(src *plan.ColDef) *plan.ColDef { } } -func _getDefaultColDefs() []*plan.ColDef { - ret := make([]*plan.ColDef, 0, len(defaultColDefs)) - for _, v := range defaultColDefs { +func _getColDefs(colDefs []*plan.ColDef) []*plan.ColDef { + ret := make([]*plan.ColDef, 0, len(colDefs)) + for _, v := range colDefs { ret = append(ret, _dupColDef(v)) } return ret } func (builder *QueryBuilder) buildUnnest(tbl *tree.TableFunction, ctx *BindContext, exprs []*plan.Expr, childId int32) (int32, error) { - colDefs := _getDefaultColDefs() + colDefs := _getColDefs(defaultColDefs) colName := findColName(tbl.Func) node := &plan.Node{ NodeType: plan.Node_FUNCTION_SCAN, diff --git a/pkg/sql/plan/utils.go b/pkg/sql/plan/utils.go index b0ebd1778d854..e858e71c4b518 100644 --- a/pkg/sql/plan/utils.go +++ b/pkg/sql/plan/utils.go @@ -1467,6 +1467,131 @@ func InitS3Param(param *tree.ExternParam) error { return nil } +func GetFilePathFromParam(param *tree.ExternParam) string { + fpath := param.Filepath + for i := 0; i < len(param.Option); i += 2 { + name := strings.ToLower(param.Option[i]) + if name == "filepath" { + fpath = param.Option[i+1] + break + } + } + + return fpath +} + +func InitStageS3Param(param *tree.ExternParam, s function.StageDef) error { + + param.ScanType = tree.S3 + param.S3Param = &tree.S3Parameter{} + + if len(s.Url.RawQuery) > 0 { + return moerr.NewBadConfig(param.Ctx, "S3 URL Query does not support in ExternParam") + } + + if s.Url.Scheme != function.S3_PROTOCOL { + return moerr.NewBadConfig(param.Ctx, "URL protocol is not S3") + } + + bucket, prefix, _, err := function.ParseS3Url(s.Url) + if err != nil { + return err + } + + var found bool + param.S3Param.Bucket = bucket + param.Filepath = prefix + + // mandatory + param.S3Param.APIKey, found = s.GetCredentials(function.PARAMKEY_AWS_KEY_ID, "") + if !found { + return moerr.NewBadConfig(param.Ctx, "Credentials %s not found", function.PARAMKEY_AWS_KEY_ID) + } + param.S3Param.APISecret, found = s.GetCredentials(function.PARAMKEY_AWS_SECRET_KEY, "") + if !found { + return moerr.NewBadConfig(param.Ctx, "Credentials %s not found", function.PARAMKEY_AWS_SECRET_KEY) + } + + param.S3Param.Region, found = s.GetCredentials(function.PARAMKEY_AWS_REGION, "") + if !found { + return moerr.NewBadConfig(param.Ctx, "Credentials %s not found", function.PARAMKEY_AWS_REGION) + } + + param.S3Param.Endpoint, found = s.GetCredentials(function.PARAMKEY_ENDPOINT, "") + if !found { + return moerr.NewBadConfig(param.Ctx, "Credentials %s not found", function.PARAMKEY_ENDPOINT) + } + + // optional + param.S3Param.Provider, _ = s.GetCredentials(function.PARAMKEY_PROVIDER, function.S3_PROVIDER_AMAZON) + param.CompressType, _ = s.GetCredentials(function.PARAMKEY_COMPRESSION, "auto") + + for i := 0; i < len(param.Option); i += 2 { + switch strings.ToLower(param.Option[i]) { + case "format": + format := strings.ToLower(param.Option[i+1]) + if format != tree.CSV && format != tree.JSONLINE { + return moerr.NewBadConfig(param.Ctx, "the format '%s' is not supported", format) + } + param.Format = format + case "jsondata": + jsondata := strings.ToLower(param.Option[i+1]) + if jsondata != tree.OBJECT && jsondata != tree.ARRAY { + return moerr.NewBadConfig(param.Ctx, "the jsondata '%s' is not supported", jsondata) + } + param.JsonData = jsondata + param.Format = tree.JSONLINE + + default: + return moerr.NewBadConfig(param.Ctx, "the keyword '%s' is not support", strings.ToLower(param.Option[i])) + } + } + + if param.Format == tree.JSONLINE && len(param.JsonData) == 0 { + return moerr.NewBadConfig(param.Ctx, "the jsondata must be specified") + } + if len(param.Format) == 0 { + param.Format = tree.CSV + } + + return nil + +} + +func InitInfileOrStageParam(param *tree.ExternParam, proc *process.Process) error { + + fpath := GetFilePathFromParam(param) + + if !strings.HasPrefix(fpath, function.STAGE_PROTOCOL+"://") { + return InitInfileParam(param) + } + + s, err := function.UrlToStageDef(fpath, proc) + if err != nil { + return err + } + + if len(s.Url.RawQuery) > 0 { + return moerr.NewBadConfig(param.Ctx, "Invalid URL: query not supported in ExternParam") + } + + if s.Url.Scheme == function.S3_PROTOCOL { + return InitStageS3Param(param, s) + } else if s.Url.Scheme == function.FILE_PROTOCOL { + + err := InitInfileParam(param) + if err != nil { + return err + } + + param.Filepath = s.Url.Path + + } else { + return moerr.NewBadConfig(param.Ctx, "invalid URL: protocol %s not supported", s.Url.Scheme) + } + + return nil +} func GetForETLWithType(param *tree.ExternParam, prefix string) (res fileservice.ETLFileService, readPath string, err error) { if param.ScanType == tree.S3 { buf := new(strings.Builder) diff --git a/test/distributed/cases/load_data/load_data_parquet.result b/test/distributed/cases/load_data/load_data_parquet.result index a8af94182e428..ead80f2675de6 100644 --- a/test/distributed/cases/load_data/load_data_parquet.result +++ b/test/distributed/cases/load_data/load_data_parquet.result @@ -42,4 +42,18 @@ xyzABCxyzABCxyzABCxyzABCxyzABCxyzABCxyzABCxyzABCxyzABCxyzABC null null xyzABCxyzABCxyzABCxyzABCxyzABCxyzABCxyzABCxyzABCxyzABCxyzABC -drop database parq; \ No newline at end of file +create table t4(id bigint not null, name varchar not null, sex bool, f32 float(5,2)); +create stage parqstage URL='file:///$resources/load_data/'; +load data infile {'filepath'='stage://parqstage/simple2.parq', 'format'='parquet'} into table t4; +select * from t4; +id name sex f32 +1 user1 false 1.0 +2 user2 false null +7 user7 false 7.0 +8 user8 true null +10 user10 false 10.0 +12 user12 false null +15 user15 false null +16 user16 true null +drop stage parqstage; +drop database parq; diff --git a/test/distributed/cases/load_data/load_data_parquet.sql b/test/distributed/cases/load_data/load_data_parquet.sql index 0cfdb116c0de2..b9c206582c008 100644 --- a/test/distributed/cases/load_data/load_data_parquet.sql +++ b/test/distributed/cases/load_data/load_data_parquet.sql @@ -16,5 +16,12 @@ create table t3(c varchar); load data infile {'filepath'='$resources/load_data/indexed_str.parq', 'format'='parquet'} into table t3; select * from t3; + +create table t4(id bigint not null, name varchar not null, sex bool, f32 float(5,2)); +create stage parqstage URL='file:///$resources/load_data/'; +load data infile {'filepath'='stage://parqstage/simple2.parq', 'format'='parquet'} into table t4; +select * from t4; +drop stage parqstage; + -- post -drop database parq; \ No newline at end of file +drop database parq; diff --git a/test/distributed/cases/snapshot/cluster/restore_cluster_table.result b/test/distributed/cases/snapshot/cluster/restore_cluster_table.result index be7de9341ea3b..46882b6369a5b 100644 --- a/test/distributed/cases/snapshot/cluster/restore_cluster_table.result +++ b/test/distributed/cases/snapshot/cluster/restore_cluster_table.result @@ -222,8 +222,8 @@ id val 3 c select * from mo_catalog.mo_stored_procedure; proc_id name creator args body db definer modified_time created_time type security_type comment character_set_client collation_connection database_collation -1 test_if_hit_elseif_first_elseif null {} begin DECLARE v1 INT; SET v1 = 5; IF v1 > 5 THEN select * from tbh1; ELSEIF v1 = 5 THEN select * from tbh2; ELSEIF v1 = 4 THEN select * from tbh2 limit 1; ELSE select * from tbh3; END IF; end procedure_test test_account 2024-07-10 07:43:56 2024-07-10 07:43:56 PROCEDURE DEFINER utf8mb4 utf8mb4_0900_ai_ci utf8mb4_0900_ai_ci -2 test_if_hit_if null {} begin DECLARE v1 INT; SET v1 = 5; IF v1 > 5 THEN select * from tbh1; ELSEIF v1 = 5 THEN select * from tbh2; ELSEIF v1 = 4 THEN select * from tbh2 limit 1; ELSE select * from tbh3; END IF; end procedure_test test_account 2024-07-10 07:43:56 2024-07-10 07:43:56 PROCEDURE DEFINER utf8mb4 utf8mb4_0900_ai_ci utf8mb4_0900_ai_ci +1 test_if_hit_elseif_first_elseif null {} begin DECLARE v1 INT; SET v1 = 5; IF v1 > 5 THEN select * from tbh1; ELSEIF v1 = 5 THEN select * from tbh2; ELSEIF v1 = 4 THEN select * from tbh2 limit 1; ELSE select * from tbh3; END IF; end procedure_test test_account 2024-08-07 09:44:26 2024-08-07 09:44:26 PROCEDURE DEFINER utf8mb4 utf8mb4_0900_ai_ci utf8mb4_0900_ai_ci +2 test_if_hit_if null {} begin DECLARE v1 INT; SET v1 = 5; IF v1 > 5 THEN select * from tbh1; ELSEIF v1 = 5 THEN select * from tbh2; ELSEIF v1 = 4 THEN select * from tbh2 limit 1; ELSE select * from tbh3; END IF; end procedure_test test_account 2024-08-07 09:44:26 2024-08-07 09:44:26 PROCEDURE DEFINER utf8mb4 utf8mb4_0900_ai_ci utf8mb4_0900_ai_ci drop database if exists udf_db2; create database udf_db2; use udf_db2; @@ -232,15 +232,15 @@ language sql as '$1 + $2'; select * from mo_catalog.mo_user_defined_function; function_id name owner args rettype body language db definer modified_time created_time type security_type comment character_set_client collation_connection database_collation -1 addab 2 [{"name": "x", "type": "int"}, {"name": "y", "type": "int"}] int $1 + $2 sql udf_db2 test_account 2024-07-10 07:43:56 2024-07-10 07:43:56 FUNCTION DEFINER utf8mb4 utf8mb4_0900_ai_ci utf8mb4_0900_ai_ci +1 addab 2 [{"name": "x", "type": "int"}, {"name": "y", "type": "int"}] int $1 + $2 sql udf_db2 test_account 2024-08-07 09:44:26 2024-08-07 09:44:26 FUNCTION DEFINER utf8mb4 utf8mb4_0900_ai_ci utf8mb4_0900_ai_ci drop stage if exists my_ext_stage; create stage my_ext_stage URL='s3://load/files/'; drop stage if exists my_ext_stage1; create stage my_ext_stage1 URL='s3://load/files/' CREDENTIALS={'AWS_KEY_ID'='1a2b3c' ,'AWS_SECRET_KEY'='4x5y6z'}; select * from mo_catalog.mo_stages; stage_id stage_name url stage_credentials stage_status created_time comment -1 my_ext_stage s3://load/files/ disabled 2024-07-10 07:43:56 -2 my_ext_stage1 s3://load/files/ *0E0591E0E0FC0806815D04D93FD4BD3F5ADB39AD disabled 2024-07-10 07:43:56 +1 my_ext_stage s3://load/files/ disabled 2024-08-07 09:44:26 +2 my_ext_stage1 s3://load/files/ AWS_KEY_ID=1a2b3c,AWS_SECRET_KEY=4x5y6z disabled 2024-08-07 09:44:26 drop database if exists test01; drop database if exists test02; drop database if exists test03; @@ -376,7 +376,7 @@ drop snapshot if exists cluster_level_snapshot; create snapshot cluster_level_snapshot for cluster; show snapshots; SNAPSHOT_NAME TIMESTAMP SNAPSHOT_LEVEL ACCOUNT_NAME DATABASE_NAME TABLE_NAME -cluster_level_snapshot 2024-07-10 07:43:56.503236 cluster +cluster_level_snapshot 2024-08-07 09:44:26.651225093 cluster use db02; alter table departments add column newcolumn int after department_id; show create table departments; @@ -447,14 +447,14 @@ drop function udf_db2.`addAB`(x int, y int); drop stage if exists my_ext_stage; select * from mo_catalog.mo_stages; stage_id stage_name url stage_credentials stage_status created_time comment -2 my_ext_stage1 s3://load/files/ *0E0591E0E0FC0806815D04D93FD4BD3F5ADB39AD disabled 2024-07-10 07:43:56 +2 my_ext_stage1 s3://load/files/ AWS_KEY_ID=1a2b3c,AWS_SECRET_KEY=4x5y6z disabled 2024-08-07 09:44:26 use udf_db2; create function `add`(x int, y int) returns int language sql as '$1 + $2'; select * from mo_catalog.mo_user_defined_function; function_id name owner args rettype body language db definer modified_time created_time type security_type comment character_set_client collation_connection database_collation -2 add 2 [{"name": "x", "type": "int"}, {"name": "y", "type": "int"}] int $1 + $2 sql udf_db2 test_account 2024-07-10 07:43:56 2024-07-10 07:43:56 FUNCTION DEFINER utf8mb4 utf8mb4_0900_ai_ci utf8mb4_0900_ai_ci +2 add 2 [{"name": "x", "type": "int"}, {"name": "y", "type": "int"}] int $1 + $2 sql udf_db2 test_account 2024-08-07 09:44:26 2024-08-07 09:44:26 FUNCTION DEFINER utf8mb4 utf8mb4_0900_ai_ci utf8mb4_0900_ai_ci drop database test01; use test02; select * from v01; @@ -543,12 +543,12 @@ select * from repub01.pri01; invalid database repub01 select * from mo_catalog.mo_stored_procedure; proc_id name creator args body db definer modified_time created_time type security_type comment character_set_client collation_connection database_collation -1 test_if_hit_elseif_first_elseif null {} begin DECLARE v1 INT; SET v1 = 5; IF v1 > 5 THEN select * from tbh1; ELSEIF v1 = 5 THEN select * from tbh2; ELSEIF v1 = 4 THEN select * from tbh2 limit 1; ELSE select * from tbh3; END IF; end procedure_test test_account 2024-07-10 07:43:56 2024-07-10 07:43:56 PROCEDURE DEFINER utf8mb4 utf8mb4_0900_ai_ci utf8mb4_0900_ai_ci -2 test_if_hit_if null {} begin DECLARE v1 INT; SET v1 = 5; IF v1 > 5 THEN select * from tbh1; ELSEIF v1 = 5 THEN select * from tbh2; ELSEIF v1 = 4 THEN select * from tbh2 limit 1; ELSE select * from tbh3; END IF; end procedure_test test_account 2024-07-10 07:43:56 2024-07-10 07:43:56 PROCEDURE DEFINER utf8mb4 utf8mb4_0900_ai_ci utf8mb4_0900_ai_ci +1 test_if_hit_elseif_first_elseif null {} begin DECLARE v1 INT; SET v1 = 5; IF v1 > 5 THEN select * from tbh1; ELSEIF v1 = 5 THEN select * from tbh2; ELSEIF v1 = 4 THEN select * from tbh2 limit 1; ELSE select * from tbh3; END IF; end procedure_test test_account 2024-08-07 09:44:26 2024-08-07 09:44:26 PROCEDURE DEFINER utf8mb4 utf8mb4_0900_ai_ci utf8mb4_0900_ai_ci +2 test_if_hit_if null {} begin DECLARE v1 INT; SET v1 = 5; IF v1 > 5 THEN select * from tbh1; ELSEIF v1 = 5 THEN select * from tbh2; ELSEIF v1 = 4 THEN select * from tbh2 limit 1; ELSE select * from tbh3; END IF; end procedure_test test_account 2024-08-07 09:44:26 2024-08-07 09:44:26 PROCEDURE DEFINER utf8mb4 utf8mb4_0900_ai_ci utf8mb4_0900_ai_ci select * from mo_catalog.mo_stages; stage_id stage_name url stage_credentials stage_status created_time comment -1 my_ext_stage s3://load/files/ disabled 2024-07-10 07:43:56 -2 my_ext_stage1 s3://load/files/ *0E0591E0E0FC0806815D04D93FD4BD3F5ADB39AD disabled 2024-07-10 07:43:56 +1 my_ext_stage s3://load/files/ disabled 2024-08-07 09:44:26 +2 my_ext_stage1 s3://load/files/ AWS_KEY_ID=1a2b3c,AWS_SECRET_KEY=4x5y6z disabled 2024-08-07 09:44:26 select * from mo_catalog.mo_user_defined_function; function_id name owner args rettype body language db definer modified_time created_time type security_type comment character_set_client collation_connection database_collation 1 addab 2 [{"name": "x", "type": "int"}, {"name": "y", "type": "int"}] int $1 + $2 sql udf_db2 test_account 2024-07-10 07:43:56 2024-07-10 07:43:56 FUNCTION DEFINER utf8mb4 utf8mb4_0900_ai_ci utf8mb4_0900_ai_ci @@ -586,8 +586,8 @@ drop database test; drop database repub02; select * from mo_catalog.mo_stored_procedure; proc_id name creator args body db definer modified_time created_time type security_type comment character_set_client collation_connection database_collation -1 test_if_hit_elseif_first_elseif null {} begin DECLARE v1 INT; SET v1 = 5; IF v1 > 5 THEN select * from tbh1; ELSEIF v1 = 5 THEN select * from tbh2; ELSEIF v1 = 4 THEN select * from tbh2 limit 1; ELSE select * from tbh3; END IF; end procedure_test test_account 2024-07-10 07:43:56 2024-07-10 07:43:56 PROCEDURE DEFINER utf8mb4 utf8mb4_0900_ai_ci utf8mb4_0900_ai_ci -2 test_if_hit_if null {} begin DECLARE v1 INT; SET v1 = 5; IF v1 > 5 THEN select * from tbh1; ELSEIF v1 = 5 THEN select * from tbh2; ELSEIF v1 = 4 THEN select * from tbh2 limit 1; ELSE select * from tbh3; END IF; end procedure_test test_account 2024-07-10 07:43:56 2024-07-10 07:43:56 PROCEDURE DEFINER utf8mb4 utf8mb4_0900_ai_ci utf8mb4_0900_ai_ci +1 test_if_hit_elseif_first_elseif null {} begin DECLARE v1 INT; SET v1 = 5; IF v1 > 5 THEN select * from tbh1; ELSEIF v1 = 5 THEN select * from tbh2; ELSEIF v1 = 4 THEN select * from tbh2 limit 1; ELSE select * from tbh3; END IF; end procedure_test test_account 2024-08-07 09:44:26 2024-08-07 09:44:26 PROCEDURE DEFINER utf8mb4 utf8mb4_0900_ai_ci utf8mb4_0900_ai_ci +2 test_if_hit_if null {} begin DECLARE v1 INT; SET v1 = 5; IF v1 > 5 THEN select * from tbh1; ELSEIF v1 = 5 THEN select * from tbh2; ELSEIF v1 = 4 THEN select * from tbh2 limit 1; ELSE select * from tbh3; END IF; end procedure_test test_account 2024-08-07 09:44:26 2024-08-07 09:44:26 PROCEDURE DEFINER utf8mb4 utf8mb4_0900_ai_ci utf8mb4_0900_ai_ci drop database if exists udf_db2; drop function `addab`(x int, y int); function addab doesn't exist @@ -617,7 +617,7 @@ drop snapshot if exists cluster_level_snapshot; create snapshot cluster_level_snapshot for cluster; show snapshots; SNAPSHOT_NAME TIMESTAMP SNAPSHOT_LEVEL ACCOUNT_NAME DATABASE_NAME TABLE_NAME -cluster_level_snapshot 2024-07-10 07:43:58.801459 cluster +cluster_level_snapshot 2024-08-07 09:44:29.17621458 cluster use mo_catalog; drop table if exists t2; create cluster table t2(a int); @@ -642,7 +642,7 @@ drop snapshot if exists cluster_level_snapshot; create snapshot cluster_level_snapshot for cluster; show snapshots; SNAPSHOT_NAME TIMESTAMP SNAPSHOT_LEVEL ACCOUNT_NAME DATABASE_NAME TABLE_NAME -cluster_level_snapshot 2024-07-10 07:43:58.991963 cluster +cluster_level_snapshot 2024-08-07 09:44:29.441979616 cluster use mo_catalog; drop table if exists t2; create cluster table t2(a int); @@ -667,7 +667,7 @@ drop snapshot if exists account_level_snapshot; create snapshot account_level_snapshot for account sys; show snapshots; SNAPSHOT_NAME TIMESTAMP SNAPSHOT_LEVEL ACCOUNT_NAME DATABASE_NAME TABLE_NAME -account_level_snapshot 2024-07-10 07:43:59.180327 account sys +account_level_snapshot 2024-08-07 09:44:29.684132745 account sys use mo_catalog; drop table if exists t2; create cluster table t2(a int); diff --git a/test/distributed/cases/snapshot/cluster_level_snapshot_restore_system_table_to_nonsys.result b/test/distributed/cases/snapshot/cluster_level_snapshot_restore_system_table_to_nonsys.result index abb7e912aa6d6..941bb4d56e1ee 100644 --- a/test/distributed/cases/snapshot/cluster_level_snapshot_restore_system_table_to_nonsys.result +++ b/test/distributed/cases/snapshot/cluster_level_snapshot_restore_system_table_to_nonsys.result @@ -19,7 +19,7 @@ drop snapshot if exists udf_dsp01; create snapshot udf_dsp01 for cluster; show snapshots; SNAPSHOT_NAME TIMESTAMP SNAPSHOT_LEVEL ACCOUNT_NAME DATABASE_NAME TABLE_NAME -udf_dsp01 2024-07-08 06:05:23.891903 cluster +udf_dsp01 2024-08-07 09:44:50.078186946 cluster create function`concatenate`(str1 varchar(255), str2 varchar(255)) returns varchar(255) language sql as '$1 + $2'; @@ -30,8 +30,8 @@ drop snapshot if exists udf_dsp02; create snapshot udf_dsp02 for cluster; show snapshots; SNAPSHOT_NAME TIMESTAMP SNAPSHOT_LEVEL ACCOUNT_NAME DATABASE_NAME TABLE_NAME -udf_dsp02 2024-07-08 06:05:23.970954 cluster -udf_dsp01 2024-07-08 06:05:23.891903 cluster +udf_dsp02 2024-08-07 09:44:50.093737665 cluster +udf_dsp01 2024-08-07 09:44:50.078186946 cluster drop database if exists udf_db2; create database udf_db2; use udf_db2; @@ -50,27 +50,27 @@ drop snapshot if exists udf_dsp03; create snapshot udf_dsp03 for cluster; show snapshots; SNAPSHOT_NAME TIMESTAMP SNAPSHOT_LEVEL ACCOUNT_NAME DATABASE_NAME TABLE_NAME -udf_dsp03 2024-07-08 06:05:24.097444 cluster -udf_dsp02 2024-07-08 06:05:23.970954 cluster -udf_dsp01 2024-07-08 06:05:23.891903 cluster +udf_dsp03 2024-08-07 09:44:50.125425407 cluster +udf_dsp02 2024-08-07 09:44:50.093737665 cluster +udf_dsp01 2024-08-07 09:44:50.078186946 cluster select * from mo_catalog.mo_user_defined_function; function_id name owner args rettype body language db definer modified_time created_time type security_type comment character_set_client collation_connection database_collation -1 addab 2 [{"name": "x", "type": "int"}, {"name": "y", "type": "int"}] int $1 + $2 sql udf_db test_account 2024-07-08 06:05:23 2024-07-08 06:05:23 FUNCTION DEFINER utf8mb4 utf8mb4_0900_ai_ci utf8mb4_0900_ai_ci -2 concatenate 2 [{"name": "str1", "type": "varchar"}, {"name": "str2", "type": "varchar"}] varchar $1 + $2 sql udf_db test_account 2024-07-08 06:05:23 2024-07-08 06:05:23 FUNCTION DEFINER utf8mb4 utf8mb4_0900_ai_ci utf8mb4_0900_ai_ci -3 subab 2 [{"name": "x", "type": "int"}, {"name": "y", "type": "int"}] int $1 - $2 sql udf_db2 test_account 2024-07-08 06:05:24 2024-07-08 06:05:24 FUNCTION DEFINER utf8mb4 utf8mb4_0900_ai_ci utf8mb4_0900_ai_ci +1 addab 2 [{"name": "x", "type": "int"}, {"name": "y", "type": "int"}] int $1 + $2 sql udf_db test_account 2024-08-07 09:44:50 2024-08-07 09:44:50 FUNCTION DEFINER utf8mb4 utf8mb4_0900_ai_ci utf8mb4_0900_ai_ci +2 concatenate 2 [{"name": "str1", "type": "varchar"}, {"name": "str2", "type": "varchar"}] varchar $1 + $2 sql udf_db test_account 2024-08-07 09:44:50 2024-08-07 09:44:50 FUNCTION DEFINER utf8mb4 utf8mb4_0900_ai_ci utf8mb4_0900_ai_ci +3 subab 2 [{"name": "x", "type": "int"}, {"name": "y", "type": "int"}] int $1 - $2 sql udf_db2 test_account 2024-08-07 09:44:50 2024-08-07 09:44:50 FUNCTION DEFINER utf8mb4 utf8mb4_0900_ai_ci utf8mb4_0900_ai_ci drop function subab(x int,y int); drop function udf_db.concatenate(str1 varchar(255), str2 varchar(255)); restore account acc01 from snapshot udf_dsp03; select * from mo_catalog.mo_user_defined_function; function_id name owner args rettype body language db definer modified_time created_time type security_type comment character_set_client collation_connection database_collation -1 addab 2 [{"name": "x", "type": "int"}, {"name": "y", "type": "int"}] int $1 + $2 sql udf_db test_account 2024-07-08 06:05:23 2024-07-08 06:05:23 FUNCTION DEFINER utf8mb4 utf8mb4_0900_ai_ci utf8mb4_0900_ai_ci -2 concatenate 2 [{"name": "str1", "type": "varchar"}, {"name": "str2", "type": "varchar"}] varchar $1 + $2 sql udf_db test_account 2024-07-08 06:05:23 2024-07-08 06:05:23 FUNCTION DEFINER utf8mb4 utf8mb4_0900_ai_ci utf8mb4_0900_ai_ci -3 subab 2 [{"name": "x", "type": "int"}, {"name": "y", "type": "int"}] int $1 - $2 sql udf_db2 test_account 2024-07-08 06:05:24 2024-07-08 06:05:24 FUNCTION DEFINER utf8mb4 utf8mb4_0900_ai_ci utf8mb4_0900_ai_ci +1 addab 2 [{"name": "x", "type": "int"}, {"name": "y", "type": "int"}] int $1 + $2 sql udf_db test_account 2024-08-07 09:44:50 2024-08-07 09:44:50 FUNCTION DEFINER utf8mb4 utf8mb4_0900_ai_ci utf8mb4_0900_ai_ci +2 concatenate 2 [{"name": "str1", "type": "varchar"}, {"name": "str2", "type": "varchar"}] varchar $1 + $2 sql udf_db test_account 2024-08-07 09:44:50 2024-08-07 09:44:50 FUNCTION DEFINER utf8mb4 utf8mb4_0900_ai_ci utf8mb4_0900_ai_ci +3 subab 2 [{"name": "x", "type": "int"}, {"name": "y", "type": "int"}] int $1 - $2 sql udf_db2 test_account 2024-08-07 09:44:50 2024-08-07 09:44:50 FUNCTION DEFINER utf8mb4 utf8mb4_0900_ai_ci utf8mb4_0900_ai_ci restore account acc01 from snapshot udf_dsp02; select * from mo_catalog.mo_user_defined_function; function_id name owner args rettype body language db definer modified_time created_time type security_type comment character_set_client collation_connection database_collation -1 addab 2 [{"name": "x", "type": "int"}, {"name": "y", "type": "int"}] int $1 + $2 sql udf_db test_account 2024-07-08 06:05:23 2024-07-08 06:05:23 FUNCTION DEFINER utf8mb4 utf8mb4_0900_ai_ci utf8mb4_0900_ai_ci -2 concatenate 2 [{"name": "str1", "type": "varchar"}, {"name": "str2", "type": "varchar"}] varchar $1 + $2 sql udf_db test_account 2024-07-08 06:05:23 2024-07-08 06:05:23 FUNCTION DEFINER utf8mb4 utf8mb4_0900_ai_ci utf8mb4_0900_ai_ci +1 addab 2 [{"name": "x", "type": "int"}, {"name": "y", "type": "int"}] int $1 + $2 sql udf_db test_account 2024-08-07 09:44:50 2024-08-07 09:44:50 FUNCTION DEFINER utf8mb4 utf8mb4_0900_ai_ci utf8mb4_0900_ai_ci +2 concatenate 2 [{"name": "str1", "type": "varchar"}, {"name": "str2", "type": "varchar"}] varchar $1 + $2 sql udf_db test_account 2024-08-07 09:44:50 2024-08-07 09:44:50 FUNCTION DEFINER utf8mb4 utf8mb4_0900_ai_ci utf8mb4_0900_ai_ci drop database udf_db; drop snapshot udf_dsp01; drop snapshot udf_dsp02; @@ -83,7 +83,7 @@ language sql as '$1 + $2'; select * from mo_catalog.mo_user_defined_function; function_id name owner args rettype body language db definer modified_time created_time type security_type comment character_set_client collation_connection database_collation -3 addab 2 [{"name": "x", "type": "int"}, {"name": "y", "type": "int"}] int $1 + $2 sql udf_db2 test_account 2024-07-08 06:05:25 2024-07-08 06:05:25 FUNCTION DEFINER utf8mb4 utf8mb4_0900_ai_ci utf8mb4_0900_ai_ci +3 addab 2 [{"name": "x", "type": "int"}, {"name": "y", "type": "int"}] int $1 + $2 sql udf_db2 test_account 2024-08-07 09:44:50 2024-08-07 09:44:50 FUNCTION DEFINER utf8mb4 utf8mb4_0900_ai_ci utf8mb4_0900_ai_ci drop snapshot if exists udf_sp04; create snapshot udf_sp04 for cluster; drop database udf_db2; @@ -92,7 +92,7 @@ function_id name owner args rettype body language db def restore account acc01 from snapshot udf_sp04; select * from mo_catalog.mo_user_defined_function; function_id name owner args rettype body language db definer modified_time created_time type security_type comment character_set_client collation_connection database_collation -3 addab 2 [{"name": "x", "type": "int"}, {"name": "y", "type": "int"}] int $1 + $2 sql udf_db2 test_account 2024-07-08 06:05:25 2024-07-08 06:05:25 FUNCTION DEFINER utf8mb4 utf8mb4_0900_ai_ci utf8mb4_0900_ai_ci +3 addab 2 [{"name": "x", "type": "int"}, {"name": "y", "type": "int"}] int $1 + $2 sql udf_db2 test_account 2024-08-07 09:44:50 2024-08-07 09:44:50 FUNCTION DEFINER utf8mb4 utf8mb4_0900_ai_ci utf8mb4_0900_ai_ci drop database udf_db2; drop snapshot udf_sp04; drop database if exists procedure_test; @@ -123,8 +123,8 @@ id val 3 c select * from mo_catalog.mo_stored_procedure; proc_id name creator args body db definer modified_time created_time type security_type comment character_set_client collation_connection database_collation -1 test_if_hit_elseif_first_elseif null {} begin DECLARE v1 INT; SET v1 = 5; IF v1 > 5 THEN select * from tbh1; ELSEIF v1 = 5 THEN select * from tbh2; ELSEIF v1 = 4 THEN select * from tbh2 limit 1; ELSE select * from tbh3; END IF; end procedure_test test_account 2024-07-08 06:05:26 2024-07-08 06:05:26 PROCEDURE DEFINER utf8mb4 utf8mb4_0900_ai_ci utf8mb4_0900_ai_ci -2 test_if_hit_if null {} begin DECLARE v1 INT; SET v1 = 5; IF v1 > 5 THEN select * from tbh1; ELSEIF v1 = 5 THEN select * from tbh2; ELSEIF v1 = 4 THEN select * from tbh2 limit 1; ELSE select * from tbh3; END IF; end procedure_test test_account 2024-07-08 06:05:26 2024-07-08 06:05:26 PROCEDURE DEFINER utf8mb4 utf8mb4_0900_ai_ci utf8mb4_0900_ai_ci +1 test_if_hit_elseif_first_elseif null {} begin DECLARE v1 INT; SET v1 = 5; IF v1 > 5 THEN select * from tbh1; ELSEIF v1 = 5 THEN select * from tbh2; ELSEIF v1 = 4 THEN select * from tbh2 limit 1; ELSE select * from tbh3; END IF; end procedure_test test_account 2024-08-07 09:44:51 2024-08-07 09:44:51 PROCEDURE DEFINER utf8mb4 utf8mb4_0900_ai_ci utf8mb4_0900_ai_ci +2 test_if_hit_if null {} begin DECLARE v1 INT; SET v1 = 5; IF v1 > 5 THEN select * from tbh1; ELSEIF v1 = 5 THEN select * from tbh2; ELSEIF v1 = 4 THEN select * from tbh2 limit 1; ELSE select * from tbh3; END IF; end procedure_test test_account 2024-08-07 09:44:51 2024-08-07 09:44:51 PROCEDURE DEFINER utf8mb4 utf8mb4_0900_ai_ci utf8mb4_0900_ai_ci drop snapshot if exists sp_sp05; create snapshot sp_sp05 for cluster; drop procedure test_if_hit_elseif_first_elseif; @@ -132,8 +132,8 @@ drop procedure test_if_hit_if; restore account acc01 from snapshot sp_sp05; select * from mo_catalog.mo_stored_procedure; proc_id name creator args body db definer modified_time created_time type security_type comment character_set_client collation_connection database_collation -1 test_if_hit_elseif_first_elseif null {} begin DECLARE v1 INT; SET v1 = 5; IF v1 > 5 THEN select * from tbh1; ELSEIF v1 = 5 THEN select * from tbh2; ELSEIF v1 = 4 THEN select * from tbh2 limit 1; ELSE select * from tbh3; END IF; end procedure_test test_account 2024-07-08 06:05:26 2024-07-08 06:05:26 PROCEDURE DEFINER utf8mb4 utf8mb4_0900_ai_ci utf8mb4_0900_ai_ci -2 test_if_hit_if null {} begin DECLARE v1 INT; SET v1 = 5; IF v1 > 5 THEN select * from tbh1; ELSEIF v1 = 5 THEN select * from tbh2; ELSEIF v1 = 4 THEN select * from tbh2 limit 1; ELSE select * from tbh3; END IF; end procedure_test test_account 2024-07-08 06:05:26 2024-07-08 06:05:26 PROCEDURE DEFINER utf8mb4 utf8mb4_0900_ai_ci utf8mb4_0900_ai_ci +1 test_if_hit_elseif_first_elseif null {} begin DECLARE v1 INT; SET v1 = 5; IF v1 > 5 THEN select * from tbh1; ELSEIF v1 = 5 THEN select * from tbh2; ELSEIF v1 = 4 THEN select * from tbh2 limit 1; ELSE select * from tbh3; END IF; end procedure_test test_account 2024-08-07 09:44:51 2024-08-07 09:44:51 PROCEDURE DEFINER utf8mb4 utf8mb4_0900_ai_ci utf8mb4_0900_ai_ci +2 test_if_hit_if null {} begin DECLARE v1 INT; SET v1 = 5; IF v1 > 5 THEN select * from tbh1; ELSEIF v1 = 5 THEN select * from tbh2; ELSEIF v1 = 4 THEN select * from tbh2 limit 1; ELSE select * from tbh3; END IF; end procedure_test test_account 2024-08-07 09:44:51 2024-08-07 09:44:51 PROCEDURE DEFINER utf8mb4 utf8mb4_0900_ai_ci utf8mb4_0900_ai_ci call test_if_hit_elseif_first_elseif(); id val 1 a @@ -174,8 +174,8 @@ id val 3 3.5 select * from mo_catalog.mo_stored_procedure; proc_id name creator args body db definer modified_time created_time type security_type comment character_set_client collation_connection database_collation -3 test_if_hit_second_elseif null {} begin DECLARE v1 INT; SET v1 = 4; IF v1 > 5 THEN select * from tbh1; ELSEIF v1 = 5 THEN select * from tbh2; ELSEIF v1 = 4 THEN select * from tbh2 limit 1; ELSE select * from tbh3; END IF; end procedure_test test_account 2024-07-08 06:05:27 2024-07-08 06:05:27 PROCEDURE DEFINER utf8mb4 utf8mb4_0900_ai_ci utf8mb4_0900_ai_ci -4 test_if_hit_else null {} begin DECLARE v1 INT; SET v1 = 3; IF v1 > 5 THEN select * from tbh1; ELSEIF v1 = 5 THEN select * from tbh2; ELSEIF v1 = 4 THEN select * from tbh2 limit 1; ELSE select * from tbh3; END IF; end procedure_test test_account 2024-07-08 06:05:27 2024-07-08 06:05:27 PROCEDURE DEFINER utf8mb4 utf8mb4_0900_ai_ci utf8mb4_0900_ai_ci +3 test_if_hit_second_elseif null {} begin DECLARE v1 INT; SET v1 = 4; IF v1 > 5 THEN select * from tbh1; ELSEIF v1 = 5 THEN select * from tbh2; ELSEIF v1 = 4 THEN select * from tbh2 limit 1; ELSE select * from tbh3; END IF; end procedure_test test_account 2024-08-07 09:44:51 2024-08-07 09:44:51 PROCEDURE DEFINER utf8mb4 utf8mb4_0900_ai_ci utf8mb4_0900_ai_ci +4 test_if_hit_else null {} begin DECLARE v1 INT; SET v1 = 3; IF v1 > 5 THEN select * from tbh1; ELSEIF v1 = 5 THEN select * from tbh2; ELSEIF v1 = 4 THEN select * from tbh2 limit 1; ELSE select * from tbh3; END IF; end procedure_test test_account 2024-08-07 09:44:51 2024-08-07 09:44:51 PROCEDURE DEFINER utf8mb4 utf8mb4_0900_ai_ci utf8mb4_0900_ai_ci drop snapshot if exists sp_sp06; create snapshot sp_sp06 for cluster; drop table tbh1; @@ -183,7 +183,7 @@ drop table tbh2; drop procedure test_if_hit_second_elseif; select * from mo_catalog.mo_stored_procedure; proc_id name creator args body db definer modified_time created_time type security_type comment character_set_client collation_connection database_collation -4 test_if_hit_else null {} begin DECLARE v1 INT; SET v1 = 3; IF v1 > 5 THEN select * from tbh1; ELSEIF v1 = 5 THEN select * from tbh2; ELSEIF v1 = 4 THEN select * from tbh2 limit 1; ELSE select * from tbh3; END IF; end procedure_test test_account 2024-07-08 06:05:27 2024-07-08 06:05:27 PROCEDURE DEFINER utf8mb4 utf8mb4_0900_ai_ci utf8mb4_0900_ai_ci +4 test_if_hit_else null {} begin DECLARE v1 INT; SET v1 = 3; IF v1 > 5 THEN select * from tbh1; ELSEIF v1 = 5 THEN select * from tbh2; ELSEIF v1 = 4 THEN select * from tbh2 limit 1; ELSE select * from tbh3; END IF; end procedure_test test_account 2024-08-07 09:44:51 2024-08-07 09:44:51 PROCEDURE DEFINER utf8mb4 utf8mb4_0900_ai_ci utf8mb4_0900_ai_ci restore account acc01 from snapshot sp_sp06; call test_if_hit_else(); id val @@ -195,8 +195,8 @@ id val 1 a select * from mo_catalog.mo_stored_procedure; proc_id name creator args body db definer modified_time created_time type security_type comment character_set_client collation_connection database_collation -3 test_if_hit_second_elseif null {} begin DECLARE v1 INT; SET v1 = 4; IF v1 > 5 THEN select * from tbh1; ELSEIF v1 = 5 THEN select * from tbh2; ELSEIF v1 = 4 THEN select * from tbh2 limit 1; ELSE select * from tbh3; END IF; end procedure_test test_account 2024-07-08 06:05:27 2024-07-08 06:05:27 PROCEDURE DEFINER utf8mb4 utf8mb4_0900_ai_ci utf8mb4_0900_ai_ci -4 test_if_hit_else null {} begin DECLARE v1 INT; SET v1 = 3; IF v1 > 5 THEN select * from tbh1; ELSEIF v1 = 5 THEN select * from tbh2; ELSEIF v1 = 4 THEN select * from tbh2 limit 1; ELSE select * from tbh3; END IF; end procedure_test test_account 2024-07-08 06:05:27 2024-07-08 06:05:27 PROCEDURE DEFINER utf8mb4 utf8mb4_0900_ai_ci utf8mb4_0900_ai_ci +3 test_if_hit_second_elseif null {} begin DECLARE v1 INT; SET v1 = 4; IF v1 > 5 THEN select * from tbh1; ELSEIF v1 = 5 THEN select * from tbh2; ELSEIF v1 = 4 THEN select * from tbh2 limit 1; ELSE select * from tbh3; END IF; end procedure_test test_account 2024-08-07 09:44:51 2024-08-07 09:44:51 PROCEDURE DEFINER utf8mb4 utf8mb4_0900_ai_ci utf8mb4_0900_ai_ci +4 test_if_hit_else null {} begin DECLARE v1 INT; SET v1 = 3; IF v1 > 5 THEN select * from tbh1; ELSEIF v1 = 5 THEN select * from tbh2; ELSEIF v1 = 4 THEN select * from tbh2 limit 1; ELSE select * from tbh3; END IF; end procedure_test test_account 2024-08-07 09:44:51 2024-08-07 09:44:51 PROCEDURE DEFINER utf8mb4 utf8mb4_0900_ai_ci utf8mb4_0900_ai_ci drop procedure test_if_hit_second_elseif; drop procedure test_if_hit_else; drop snapshot sp_sp06; @@ -206,20 +206,20 @@ drop stage if exists my_ext_stage1; create stage my_ext_stage1 URL='s3://load/files/' CREDENTIALS={'AWS_KEY_ID'='1a2b3c' ,'AWS_SECRET_KEY'='4x5y6z'}; select * from mo_catalog.mo_stages; stage_id stage_name url stage_credentials stage_status created_time comment -1 my_ext_stage s3://load/files/ disabled 2024-07-08 06:05:27 -2 my_ext_stage1 s3://load/files/ *0E0591E0E0FC0806815D04D93FD4BD3F5ADB39AD disabled 2024-07-08 06:05:27 +1 my_ext_stage s3://load/files/ disabled 2024-08-07 09:44:51 +2 my_ext_stage1 s3://load/files/ AWS_KEY_ID=1a2b3c,AWS_SECRET_KEY=4x5y6z disabled 2024-08-07 09:44:51 drop snapshot if exists stage_sp01; create snapshot stage_sp01 for cluster; alter stage my_ext_stage1 SET URL='s3://load/files2/'; drop stage my_ext_stage; select * from mo_catalog.mo_stages; stage_id stage_name url stage_credentials stage_status created_time comment -2 my_ext_stage1 s3://load/files2/ *0E0591E0E0FC0806815D04D93FD4BD3F5ADB39AD disabled 2024-07-08 06:05:27 +2 my_ext_stage1 s3://load/files2/ AWS_KEY_ID=1a2b3c,AWS_SECRET_KEY=4x5y6z disabled 2024-08-07 09:44:51 restore account acc01 from snapshot stage_sp01; select * from mo_catalog.mo_stages; stage_id stage_name url stage_credentials stage_status created_time comment -1 my_ext_stage s3://load/files/ disabled 2024-07-08 06:05:27 -2 my_ext_stage1 s3://load/files/ *0E0591E0E0FC0806815D04D93FD4BD3F5ADB39AD disabled 2024-07-08 06:05:27 +1 my_ext_stage s3://load/files/ disabled 2024-08-07 09:44:51 +2 my_ext_stage1 s3://load/files/ AWS_KEY_ID=1a2b3c,AWS_SECRET_KEY=4x5y6z disabled 2024-08-07 09:44:51 drop stage my_ext_stage; drop stage my_ext_stage1; drop snapshot stage_sp01; @@ -703,4 +703,4 @@ drop role role_r1, role_r2, role_r3; drop user role_u1, role_u2, role_u3; drop snapshot sp01; drop account acc01; -set global enable_privilege_cache = on; \ No newline at end of file +set global enable_privilege_cache = on; diff --git a/test/distributed/cases/snapshot/nonsys_restore_system_table_to_nonsys_account.result b/test/distributed/cases/snapshot/nonsys_restore_system_table_to_nonsys_account.result index 4d7e2e9599021..5a3b421055360 100644 --- a/test/distributed/cases/snapshot/nonsys_restore_system_table_to_nonsys_account.result +++ b/test/distributed/cases/snapshot/nonsys_restore_system_table_to_nonsys_account.result @@ -19,7 +19,7 @@ drop snapshot if exists udf_dsp01; create snapshot udf_dsp01 for account acc01; show snapshots; SNAPSHOT_NAME TIMESTAMP SNAPSHOT_LEVEL ACCOUNT_NAME DATABASE_NAME TABLE_NAME -udf_dsp01 2024-05-31 03:26:32.785853 account acc01 +udf_dsp01 2024-08-07 09:45:13.242581157 account acc01 create function`concatenate`(str1 varchar(255), str2 varchar(255)) returns varchar(255) language sql as '$1 + $2'; @@ -30,8 +30,8 @@ drop snapshot if exists udf_dsp02; create snapshot udf_dsp02 for account acc01; show snapshots; SNAPSHOT_NAME TIMESTAMP SNAPSHOT_LEVEL ACCOUNT_NAME DATABASE_NAME TABLE_NAME -udf_dsp02 2024-05-31 03:26:32.842895 account acc01 -udf_dsp01 2024-05-31 03:26:32.785853 account acc01 +udf_dsp02 2024-08-07 09:45:13.279044466 account acc01 +udf_dsp01 2024-08-07 09:45:13.242581157 account acc01 drop database if exists udf_db2; create database udf_db2; use udf_db2; @@ -50,37 +50,37 @@ drop snapshot if exists udf_dsp03; create snapshot udf_dsp03 for account acc01; show snapshots; SNAPSHOT_NAME TIMESTAMP SNAPSHOT_LEVEL ACCOUNT_NAME DATABASE_NAME TABLE_NAME -udf_dsp03 2024-05-31 03:26:32.993994 account acc01 -udf_dsp02 2024-05-31 03:26:32.842895 account acc01 -udf_dsp01 2024-05-31 03:26:32.785853 account acc01 +udf_dsp03 2024-08-07 09:45:13.313005033 account acc01 +udf_dsp02 2024-08-07 09:45:13.279044466 account acc01 +udf_dsp01 2024-08-07 09:45:13.242581157 account acc01 select * from mo_catalog.mo_user_defined_function; function_id name owner args rettype body language db definer modified_time created_time type security_type comment character_set_client collation_connection database_collation -1 addab 2 [{"name": "x", "type": "int"}, {"name": "y", "type": "int"}] int $1 + $2 sql udf_db test_account 2024-05-31 03:26:32 2024-05-31 03:26:32 FUNCTION DEFINER utf8mb4 utf8mb4_0900_ai_ci utf8mb4_0900_ai_ci -2 concatenate 2 [{"name": "str1", "type": "varchar"}, {"name": "str2", "type": "varchar"}] varchar $1 + $2 sql udf_db test_account 2024-05-31 03:26:32 2024-05-31 03:26:32 FUNCTION DEFINER utf8mb4 utf8mb4_0900_ai_ci utf8mb4_0900_ai_ci -3 subab 2 [{"name": "x", "type": "int"}, {"name": "y", "type": "int"}] int $1 - $2 sql udf_db2 test_account 2024-05-31 03:26:32 2024-05-31 03:26:32 FUNCTION DEFINER utf8mb4 utf8mb4_0900_ai_ci utf8mb4_0900_ai_ci +1 addab 2 [{"name": "x", "type": "int"}, {"name": "y", "type": "int"}] int $1 + $2 sql udf_db test_account 2024-08-07 09:45:13 2024-08-07 09:45:13 FUNCTION DEFINER utf8mb4 utf8mb4_0900_ai_ci utf8mb4_0900_ai_ci +2 concatenate 2 [{"name": "str1", "type": "varchar"}, {"name": "str2", "type": "varchar"}] varchar $1 + $2 sql udf_db test_account 2024-08-07 09:45:13 2024-08-07 09:45:13 FUNCTION DEFINER utf8mb4 utf8mb4_0900_ai_ci utf8mb4_0900_ai_ci +3 subab 2 [{"name": "x", "type": "int"}, {"name": "y", "type": "int"}] int $1 - $2 sql udf_db2 test_account 2024-08-07 09:45:13 2024-08-07 09:45:13 FUNCTION DEFINER utf8mb4 utf8mb4_0900_ai_ci utf8mb4_0900_ai_ci drop function subab(x int,y int); drop function udf_db.concatenate(str1 varchar(255), str2 varchar(255)); restore account acc01 from snapshot udf_dsp03; select * from mo_catalog.mo_user_defined_function; function_id name owner args rettype body language db definer modified_time created_time type security_type comment character_set_client collation_connection database_collation -1 addab 2 [{"name": "x", "type": "int"}, {"name": "y", "type": "int"}] int $1 + $2 sql udf_db test_account 2024-05-31 03:26:32 2024-05-31 03:26:32 FUNCTION DEFINER utf8mb4 utf8mb4_0900_ai_ci utf8mb4_0900_ai_ci -2 concatenate 2 [{"name": "str1", "type": "varchar"}, {"name": "str2", "type": "varchar"}] varchar $1 + $2 sql udf_db test_account 2024-05-31 03:26:32 2024-05-31 03:26:32 FUNCTION DEFINER utf8mb4 utf8mb4_0900_ai_ci utf8mb4_0900_ai_ci -3 subab 2 [{"name": "x", "type": "int"}, {"name": "y", "type": "int"}] int $1 - $2 sql udf_db2 test_account 2024-05-31 03:26:32 2024-05-31 03:26:32 FUNCTION DEFINER utf8mb4 utf8mb4_0900_ai_ci utf8mb4_0900_ai_ci +1 addab 2 [{"name": "x", "type": "int"}, {"name": "y", "type": "int"}] int $1 + $2 sql udf_db test_account 2024-08-07 09:45:13 2024-08-07 09:45:13 FUNCTION DEFINER utf8mb4 utf8mb4_0900_ai_ci utf8mb4_0900_ai_ci +2 concatenate 2 [{"name": "str1", "type": "varchar"}, {"name": "str2", "type": "varchar"}] varchar $1 + $2 sql udf_db test_account 2024-08-07 09:45:13 2024-08-07 09:45:13 FUNCTION DEFINER utf8mb4 utf8mb4_0900_ai_ci utf8mb4_0900_ai_ci +3 subab 2 [{"name": "x", "type": "int"}, {"name": "y", "type": "int"}] int $1 - $2 sql udf_db2 test_account 2024-08-07 09:45:13 2024-08-07 09:45:13 FUNCTION DEFINER utf8mb4 utf8mb4_0900_ai_ci utf8mb4_0900_ai_ci show snapshots; SNAPSHOT_NAME TIMESTAMP SNAPSHOT_LEVEL ACCOUNT_NAME DATABASE_NAME TABLE_NAME -udf_dsp03 2024-05-31 03:26:32.993994 account acc01 -udf_dsp02 2024-05-31 03:26:32.842895 account acc01 -udf_dsp01 2024-05-31 03:26:32.785853 account acc01 +udf_dsp03 2024-08-07 09:45:13.313005033 account acc01 +udf_dsp02 2024-08-07 09:45:13.279044466 account acc01 +udf_dsp01 2024-08-07 09:45:13.242581157 account acc01 restore account acc01 from snapshot udf_dsp02; select * from mo_catalog.mo_user_defined_function; function_id name owner args rettype body language db definer modified_time created_time type security_type comment character_set_client collation_connection database_collation -1 addab 2 [{"name": "x", "type": "int"}, {"name": "y", "type": "int"}] int $1 + $2 sql udf_db test_account 2024-05-31 03:26:32 2024-05-31 03:26:32 FUNCTION DEFINER utf8mb4 utf8mb4_0900_ai_ci utf8mb4_0900_ai_ci -2 concatenate 2 [{"name": "str1", "type": "varchar"}, {"name": "str2", "type": "varchar"}] varchar $1 + $2 sql udf_db test_account 2024-05-31 03:26:32 2024-05-31 03:26:32 FUNCTION DEFINER utf8mb4 utf8mb4_0900_ai_ci utf8mb4_0900_ai_ci +1 addab 2 [{"name": "x", "type": "int"}, {"name": "y", "type": "int"}] int $1 + $2 sql udf_db test_account 2024-08-07 09:45:13 2024-08-07 09:45:13 FUNCTION DEFINER utf8mb4 utf8mb4_0900_ai_ci utf8mb4_0900_ai_ci +2 concatenate 2 [{"name": "str1", "type": "varchar"}, {"name": "str2", "type": "varchar"}] varchar $1 + $2 sql udf_db test_account 2024-08-07 09:45:13 2024-08-07 09:45:13 FUNCTION DEFINER utf8mb4 utf8mb4_0900_ai_ci utf8mb4_0900_ai_ci show snapshots; SNAPSHOT_NAME TIMESTAMP SNAPSHOT_LEVEL ACCOUNT_NAME DATABASE_NAME TABLE_NAME -udf_dsp03 2024-05-31 03:26:32.993994 account acc01 -udf_dsp02 2024-05-31 03:26:32.842895 account acc01 -udf_dsp01 2024-05-31 03:26:32.785853 account acc01 +udf_dsp03 2024-08-07 09:45:13.313005033 account acc01 +udf_dsp02 2024-08-07 09:45:13.279044466 account acc01 +udf_dsp01 2024-08-07 09:45:13.242581157 account acc01 drop snapshot udf_dsp01; drop snapshot udf_dsp02; drop snapshot udf_dsp03; @@ -93,7 +93,7 @@ language sql as '$1 + $2'; select * from mo_catalog.mo_user_defined_function; function_id name owner args rettype body language db definer modified_time created_time type security_type comment character_set_client collation_connection database_collation -3 addab 2 [{"name": "x", "type": "int"}, {"name": "y", "type": "int"}] int $1 + $2 sql udf_db2 test_account 2024-05-31 03:26:34 2024-05-31 03:26:34 FUNCTION DEFINER utf8mb4 utf8mb4_0900_ai_ci utf8mb4_0900_ai_ci +3 addab 2 [{"name": "x", "type": "int"}, {"name": "y", "type": "int"}] int $1 + $2 sql udf_db2 test_account 2024-08-07 09:45:13 2024-08-07 09:45:13 FUNCTION DEFINER utf8mb4 utf8mb4_0900_ai_ci utf8mb4_0900_ai_ci drop snapshot if exists udf_sp04; create snapshot udf_sp04 for account acc01; drop database udf_db2; @@ -102,7 +102,7 @@ function_id name owner args rettype body language db def restore account acc01 from snapshot udf_sp04; select * from mo_catalog.mo_user_defined_function; function_id name owner args rettype body language db definer modified_time created_time type security_type comment character_set_client collation_connection database_collation -3 addab 2 [{"name": "x", "type": "int"}, {"name": "y", "type": "int"}] int $1 + $2 sql udf_db2 test_account 2024-05-31 03:26:34 2024-05-31 03:26:34 FUNCTION DEFINER utf8mb4 utf8mb4_0900_ai_ci utf8mb4_0900_ai_ci +3 addab 2 [{"name": "x", "type": "int"}, {"name": "y", "type": "int"}] int $1 + $2 sql udf_db2 test_account 2024-08-07 09:45:13 2024-08-07 09:45:13 FUNCTION DEFINER utf8mb4 utf8mb4_0900_ai_ci utf8mb4_0900_ai_ci drop database udf_db2; drop snapshot udf_sp04; drop database if exists procedure_test; @@ -133,8 +133,8 @@ id val 3 c select * from mo_catalog.mo_stored_procedure; proc_id name creator args body db definer modified_time created_time type security_type comment character_set_client collation_connection database_collation -1 test_if_hit_elseif_first_elseif null {} begin DECLARE v1 INT; SET v1 = 5; IF v1 > 5 THEN select * from tbh1; ELSEIF v1 = 5 THEN select * from tbh2; ELSEIF v1 = 4 THEN select * from tbh2 limit 1; ELSE select * from tbh3; END IF; end procedure_test test_account 2024-05-31 03:26:35 2024-05-31 03:26:35 PROCEDURE DEFINER utf8mb4 utf8mb4_0900_ai_ci utf8mb4_0900_ai_ci -2 test_if_hit_if null {} begin DECLARE v1 INT; SET v1 = 5; IF v1 > 5 THEN select * from tbh1; ELSEIF v1 = 5 THEN select * from tbh2; ELSEIF v1 = 4 THEN select * from tbh2 limit 1; ELSE select * from tbh3; END IF; end procedure_test test_account 2024-05-31 03:26:35 2024-05-31 03:26:35 PROCEDURE DEFINER utf8mb4 utf8mb4_0900_ai_ci utf8mb4_0900_ai_ci +1 test_if_hit_elseif_first_elseif null {} begin DECLARE v1 INT; SET v1 = 5; IF v1 > 5 THEN select * from tbh1; ELSEIF v1 = 5 THEN select * from tbh2; ELSEIF v1 = 4 THEN select * from tbh2 limit 1; ELSE select * from tbh3; END IF; end procedure_test test_account 2024-08-07 09:45:14 2024-08-07 09:45:14 PROCEDURE DEFINER utf8mb4 utf8mb4_0900_ai_ci utf8mb4_0900_ai_ci +2 test_if_hit_if null {} begin DECLARE v1 INT; SET v1 = 5; IF v1 > 5 THEN select * from tbh1; ELSEIF v1 = 5 THEN select * from tbh2; ELSEIF v1 = 4 THEN select * from tbh2 limit 1; ELSE select * from tbh3; END IF; end procedure_test test_account 2024-08-07 09:45:14 2024-08-07 09:45:14 PROCEDURE DEFINER utf8mb4 utf8mb4_0900_ai_ci utf8mb4_0900_ai_ci drop snapshot if exists sp_sp05; create snapshot sp_sp05 for account acc01; drop procedure test_if_hit_elseif_first_elseif; @@ -144,8 +144,8 @@ proc_id name creator args body db definer modified_time restore account acc01 from snapshot sp_sp05; select * from mo_catalog.mo_stored_procedure; proc_id name creator args body db definer modified_time created_time type security_type comment character_set_client collation_connection database_collation -1 test_if_hit_elseif_first_elseif null {} begin DECLARE v1 INT; SET v1 = 5; IF v1 > 5 THEN select * from tbh1; ELSEIF v1 = 5 THEN select * from tbh2; ELSEIF v1 = 4 THEN select * from tbh2 limit 1; ELSE select * from tbh3; END IF; end procedure_test test_account 2024-05-31 03:26:35 2024-05-31 03:26:35 PROCEDURE DEFINER utf8mb4 utf8mb4_0900_ai_ci utf8mb4_0900_ai_ci -2 test_if_hit_if null {} begin DECLARE v1 INT; SET v1 = 5; IF v1 > 5 THEN select * from tbh1; ELSEIF v1 = 5 THEN select * from tbh2; ELSEIF v1 = 4 THEN select * from tbh2 limit 1; ELSE select * from tbh3; END IF; end procedure_test test_account 2024-05-31 03:26:35 2024-05-31 03:26:35 PROCEDURE DEFINER utf8mb4 utf8mb4_0900_ai_ci utf8mb4_0900_ai_ci +1 test_if_hit_elseif_first_elseif null {} begin DECLARE v1 INT; SET v1 = 5; IF v1 > 5 THEN select * from tbh1; ELSEIF v1 = 5 THEN select * from tbh2; ELSEIF v1 = 4 THEN select * from tbh2 limit 1; ELSE select * from tbh3; END IF; end procedure_test test_account 2024-08-07 09:45:14 2024-08-07 09:45:14 PROCEDURE DEFINER utf8mb4 utf8mb4_0900_ai_ci utf8mb4_0900_ai_ci +2 test_if_hit_if null {} begin DECLARE v1 INT; SET v1 = 5; IF v1 > 5 THEN select * from tbh1; ELSEIF v1 = 5 THEN select * from tbh2; ELSEIF v1 = 4 THEN select * from tbh2 limit 1; ELSE select * from tbh3; END IF; end procedure_test test_account 2024-08-07 09:45:14 2024-08-07 09:45:14 PROCEDURE DEFINER utf8mb4 utf8mb4_0900_ai_ci utf8mb4_0900_ai_ci call test_if_hit_elseif_first_elseif(); id val 1 a @@ -186,8 +186,8 @@ id val 3 3.5 select * from mo_catalog.mo_stored_procedure; proc_id name creator args body db definer modified_time created_time type security_type comment character_set_client collation_connection database_collation -3 test_if_hit_second_elseif null {} begin DECLARE v1 INT; SET v1 = 4; IF v1 > 5 THEN select * from tbh1; ELSEIF v1 = 5 THEN select * from tbh2; ELSEIF v1 = 4 THEN select * from tbh2 limit 1; ELSE select * from tbh3; END IF; end procedure_test test_account 2024-05-31 03:26:36 2024-05-31 03:26:36 PROCEDURE DEFINER utf8mb4 utf8mb4_0900_ai_ci utf8mb4_0900_ai_ci -4 test_if_hit_else null {} begin DECLARE v1 INT; SET v1 = 3; IF v1 > 5 THEN select * from tbh1; ELSEIF v1 = 5 THEN select * from tbh2; ELSEIF v1 = 4 THEN select * from tbh2 limit 1; ELSE select * from tbh3; END IF; end procedure_test test_account 2024-05-31 03:26:36 2024-05-31 03:26:36 PROCEDURE DEFINER utf8mb4 utf8mb4_0900_ai_ci utf8mb4_0900_ai_ci +3 test_if_hit_second_elseif null {} begin DECLARE v1 INT; SET v1 = 4; IF v1 > 5 THEN select * from tbh1; ELSEIF v1 = 5 THEN select * from tbh2; ELSEIF v1 = 4 THEN select * from tbh2 limit 1; ELSE select * from tbh3; END IF; end procedure_test test_account 2024-08-07 09:45:14 2024-08-07 09:45:14 PROCEDURE DEFINER utf8mb4 utf8mb4_0900_ai_ci utf8mb4_0900_ai_ci +4 test_if_hit_else null {} begin DECLARE v1 INT; SET v1 = 3; IF v1 > 5 THEN select * from tbh1; ELSEIF v1 = 5 THEN select * from tbh2; ELSEIF v1 = 4 THEN select * from tbh2 limit 1; ELSE select * from tbh3; END IF; end procedure_test test_account 2024-08-07 09:45:14 2024-08-07 09:45:14 PROCEDURE DEFINER utf8mb4 utf8mb4_0900_ai_ci utf8mb4_0900_ai_ci drop snapshot if exists sp_sp06; create snapshot sp_sp06 for account acc01; drop table tbh1; @@ -195,7 +195,7 @@ drop table tbh2; drop procedure test_if_hit_second_elseif; select * from mo_catalog.mo_stored_procedure; proc_id name creator args body db definer modified_time created_time type security_type comment character_set_client collation_connection database_collation -4 test_if_hit_else null {} begin DECLARE v1 INT; SET v1 = 3; IF v1 > 5 THEN select * from tbh1; ELSEIF v1 = 5 THEN select * from tbh2; ELSEIF v1 = 4 THEN select * from tbh2 limit 1; ELSE select * from tbh3; END IF; end procedure_test test_account 2024-05-31 03:26:36 2024-05-31 03:26:36 PROCEDURE DEFINER utf8mb4 utf8mb4_0900_ai_ci utf8mb4_0900_ai_ci +4 test_if_hit_else null {} begin DECLARE v1 INT; SET v1 = 3; IF v1 > 5 THEN select * from tbh1; ELSEIF v1 = 5 THEN select * from tbh2; ELSEIF v1 = 4 THEN select * from tbh2 limit 1; ELSE select * from tbh3; END IF; end procedure_test test_account 2024-08-07 09:45:14 2024-08-07 09:45:14 PROCEDURE DEFINER utf8mb4 utf8mb4_0900_ai_ci utf8mb4_0900_ai_ci restore account acc01 from snapshot sp_sp06; call test_if_hit_else(); id val @@ -207,8 +207,8 @@ id val 1 a select * from mo_catalog.mo_stored_procedure; proc_id name creator args body db definer modified_time created_time type security_type comment character_set_client collation_connection database_collation -3 test_if_hit_second_elseif null {} begin DECLARE v1 INT; SET v1 = 4; IF v1 > 5 THEN select * from tbh1; ELSEIF v1 = 5 THEN select * from tbh2; ELSEIF v1 = 4 THEN select * from tbh2 limit 1; ELSE select * from tbh3; END IF; end procedure_test test_account 2024-05-31 03:26:36 2024-05-31 03:26:36 PROCEDURE DEFINER utf8mb4 utf8mb4_0900_ai_ci utf8mb4_0900_ai_ci -4 test_if_hit_else null {} begin DECLARE v1 INT; SET v1 = 3; IF v1 > 5 THEN select * from tbh1; ELSEIF v1 = 5 THEN select * from tbh2; ELSEIF v1 = 4 THEN select * from tbh2 limit 1; ELSE select * from tbh3; END IF; end procedure_test test_account 2024-05-31 03:26:36 2024-05-31 03:26:36 PROCEDURE DEFINER utf8mb4 utf8mb4_0900_ai_ci utf8mb4_0900_ai_ci +3 test_if_hit_second_elseif null {} begin DECLARE v1 INT; SET v1 = 4; IF v1 > 5 THEN select * from tbh1; ELSEIF v1 = 5 THEN select * from tbh2; ELSEIF v1 = 4 THEN select * from tbh2 limit 1; ELSE select * from tbh3; END IF; end procedure_test test_account 2024-08-07 09:45:14 2024-08-07 09:45:14 PROCEDURE DEFINER utf8mb4 utf8mb4_0900_ai_ci utf8mb4_0900_ai_ci +4 test_if_hit_else null {} begin DECLARE v1 INT; SET v1 = 3; IF v1 > 5 THEN select * from tbh1; ELSEIF v1 = 5 THEN select * from tbh2; ELSEIF v1 = 4 THEN select * from tbh2 limit 1; ELSE select * from tbh3; END IF; end procedure_test test_account 2024-08-07 09:45:14 2024-08-07 09:45:14 PROCEDURE DEFINER utf8mb4 utf8mb4_0900_ai_ci utf8mb4_0900_ai_ci drop snapshot sp_sp06; drop procedure test_if_hit_second_elseif; drop procedure test_if_hit_else; @@ -218,20 +218,20 @@ drop stage if exists my_ext_stage1; create stage my_ext_stage1 URL='s3://load/files/' CREDENTIALS={'AWS_KEY_ID'='1a2b3c' ,'AWS_SECRET_KEY'='4x5y6z'}; select * from mo_catalog.mo_stages; stage_id stage_name url stage_credentials stage_status created_time comment -1 my_ext_stage s3://load/files/ disabled 2024-05-31 03:26:37 -2 my_ext_stage1 s3://load/files/ *0E0591E0E0FC0806815D04D93FD4BD3F5ADB39AD disabled 2024-05-31 03:26:37 +1 my_ext_stage s3://load/files/ disabled 2024-08-07 09:45:14 +2 my_ext_stage1 s3://load/files/ AWS_KEY_ID=1a2b3c,AWS_SECRET_KEY=4x5y6z disabled 2024-08-07 09:45:14 drop snapshot if exists stage_sp01; create snapshot stage_sp01 for account acc01; alter stage my_ext_stage1 SET URL='s3://load/files2/'; drop stage my_ext_stage; select * from mo_catalog.mo_stages; stage_id stage_name url stage_credentials stage_status created_time comment -2 my_ext_stage1 s3://load/files2/ *0E0591E0E0FC0806815D04D93FD4BD3F5ADB39AD disabled 2024-05-31 03:26:37 +2 my_ext_stage1 s3://load/files2/ AWS_KEY_ID=1a2b3c,AWS_SECRET_KEY=4x5y6z disabled 2024-08-07 09:45:14 restore account acc01 from snapshot stage_sp01; select * from mo_catalog.mo_stages; stage_id stage_name url stage_credentials stage_status created_time comment -1 my_ext_stage s3://load/files/ disabled 2024-05-31 03:26:37 -2 my_ext_stage1 s3://load/files/ *0E0591E0E0FC0806815D04D93FD4BD3F5ADB39AD disabled 2024-05-31 03:26:37 +1 my_ext_stage s3://load/files/ disabled 2024-08-07 09:45:14 +2 my_ext_stage1 s3://load/files/ AWS_KEY_ID=1a2b3c,AWS_SECRET_KEY=4x5y6z disabled 2024-08-07 09:45:14 drop snapshot stage_sp01; drop stage my_ext_stage; drop stage my_ext_stage1; diff --git a/test/distributed/cases/snapshot/sys_restore_system_table_to_newnonsys_account.result b/test/distributed/cases/snapshot/sys_restore_system_table_to_newnonsys_account.result index 95dcc74594d71..c12b36d384e89 100644 --- a/test/distributed/cases/snapshot/sys_restore_system_table_to_newnonsys_account.result +++ b/test/distributed/cases/snapshot/sys_restore_system_table_to_newnonsys_account.result @@ -21,7 +21,7 @@ drop snapshot if exists udf_dsp01; create snapshot udf_dsp01 for account acc01; show snapshots; SNAPSHOT_NAME TIMESTAMP SNAPSHOT_LEVEL ACCOUNT_NAME DATABASE_NAME TABLE_NAME -udf_dsp01 2024-05-31 03:15:09.421606 account acc01 +udf_dsp01 2024-08-07 09:45:39.337131935 account acc01 create function`concatenate`(str1 varchar(255), str2 varchar(255)) returns varchar(255) language sql as '$1 + $2'; @@ -32,8 +32,8 @@ drop snapshot if exists udf_dsp02; create snapshot udf_dsp02 for account acc01; show snapshots; SNAPSHOT_NAME TIMESTAMP SNAPSHOT_LEVEL ACCOUNT_NAME DATABASE_NAME TABLE_NAME -udf_dsp02 2024-05-31 03:15:09.487947 account acc01 -udf_dsp01 2024-05-31 03:15:09.421606 account acc01 +udf_dsp02 2024-08-07 09:45:39.384962231 account acc01 +udf_dsp01 2024-08-07 09:45:39.337131935 account acc01 drop database if exists udf_db2; create database udf_db2; use udf_db2; @@ -52,27 +52,27 @@ drop snapshot if exists udf_dsp03; create snapshot udf_dsp03 for account acc01; show snapshots; SNAPSHOT_NAME TIMESTAMP SNAPSHOT_LEVEL ACCOUNT_NAME DATABASE_NAME TABLE_NAME -udf_dsp03 2024-05-31 03:15:09.68859 account acc01 -udf_dsp02 2024-05-31 03:15:09.487947 account acc01 -udf_dsp01 2024-05-31 03:15:09.421606 account acc01 +udf_dsp03 2024-08-07 09:45:40.101171489 account acc01 +udf_dsp02 2024-08-07 09:45:39.384962231 account acc01 +udf_dsp01 2024-08-07 09:45:39.337131935 account acc01 select * from mo_catalog.mo_user_defined_function; function_id name owner args rettype body language db definer modified_time created_time type security_type comment character_set_client collation_connection database_collation -1 addab 2 [{"name": "x", "type": "int"}, {"name": "y", "type": "int"}] int $1 + $2 sql udf_db test_account 2024-05-31 03:15:09 2024-05-31 03:15:09 FUNCTION DEFINER utf8mb4 utf8mb4_0900_ai_ci utf8mb4_0900_ai_ci -2 concatenate 2 [{"name": "str1", "type": "varchar"}, {"name": "str2", "type": "varchar"}] varchar $1 + $2 sql udf_db test_account 2024-05-31 03:15:09 2024-05-31 03:15:09 FUNCTION DEFINER utf8mb4 utf8mb4_0900_ai_ci utf8mb4_0900_ai_ci -3 subab 2 [{"name": "x", "type": "int"}, {"name": "y", "type": "int"}] int $1 - $2 sql udf_db2 test_account 2024-05-31 03:15:09 2024-05-31 03:15:09 FUNCTION DEFINER utf8mb4 utf8mb4_0900_ai_ci utf8mb4_0900_ai_ci +1 addab 2 [{"name": "x", "type": "int"}, {"name": "y", "type": "int"}] int $1 + $2 sql udf_db test_account 2024-08-07 09:45:39 2024-08-07 09:45:39 FUNCTION DEFINER utf8mb4 utf8mb4_0900_ai_ci utf8mb4_0900_ai_ci +2 concatenate 2 [{"name": "str1", "type": "varchar"}, {"name": "str2", "type": "varchar"}] varchar $1 + $2 sql udf_db test_account 2024-08-07 09:45:39 2024-08-07 09:45:39 FUNCTION DEFINER utf8mb4 utf8mb4_0900_ai_ci utf8mb4_0900_ai_ci +3 subab 2 [{"name": "x", "type": "int"}, {"name": "y", "type": "int"}] int $1 - $2 sql udf_db2 test_account 2024-08-07 09:45:40 2024-08-07 09:45:40 FUNCTION DEFINER utf8mb4 utf8mb4_0900_ai_ci utf8mb4_0900_ai_ci drop function subab(x int,y int); drop function udf_db.concatenate(str1 varchar(255), str2 varchar(255)); restore account acc01 from snapshot udf_dsp03 to account acc02; select * from mo_catalog.mo_user_defined_function; function_id name owner args rettype body language db definer modified_time created_time type security_type comment character_set_client collation_connection database_collation -1 addab 2 [{"name": "x", "type": "int"}, {"name": "y", "type": "int"}] int $1 + $2 sql udf_db test_account 2024-05-31 03:15:09 2024-05-31 03:15:09 FUNCTION DEFINER utf8mb4 utf8mb4_0900_ai_ci utf8mb4_0900_ai_ci -2 concatenate 2 [{"name": "str1", "type": "varchar"}, {"name": "str2", "type": "varchar"}] varchar $1 + $2 sql udf_db test_account 2024-05-31 03:15:09 2024-05-31 03:15:09 FUNCTION DEFINER utf8mb4 utf8mb4_0900_ai_ci utf8mb4_0900_ai_ci -3 subab 2 [{"name": "x", "type": "int"}, {"name": "y", "type": "int"}] int $1 - $2 sql udf_db2 test_account 2024-05-31 03:15:09 2024-05-31 03:15:09 FUNCTION DEFINER utf8mb4 utf8mb4_0900_ai_ci utf8mb4_0900_ai_ci +1 addab 2 [{"name": "x", "type": "int"}, {"name": "y", "type": "int"}] int $1 + $2 sql udf_db test_account 2024-08-07 09:45:39 2024-08-07 09:45:39 FUNCTION DEFINER utf8mb4 utf8mb4_0900_ai_ci utf8mb4_0900_ai_ci +2 concatenate 2 [{"name": "str1", "type": "varchar"}, {"name": "str2", "type": "varchar"}] varchar $1 + $2 sql udf_db test_account 2024-08-07 09:45:39 2024-08-07 09:45:39 FUNCTION DEFINER utf8mb4 utf8mb4_0900_ai_ci utf8mb4_0900_ai_ci +3 subab 2 [{"name": "x", "type": "int"}, {"name": "y", "type": "int"}] int $1 - $2 sql udf_db2 test_account 2024-08-07 09:45:40 2024-08-07 09:45:40 FUNCTION DEFINER utf8mb4 utf8mb4_0900_ai_ci utf8mb4_0900_ai_ci restore account acc01 from snapshot udf_dsp02 to account acc02; select * from mo_catalog.mo_user_defined_function; function_id name owner args rettype body language db definer modified_time created_time type security_type comment character_set_client collation_connection database_collation -1 addab 2 [{"name": "x", "type": "int"}, {"name": "y", "type": "int"}] int $1 + $2 sql udf_db test_account 2024-05-31 03:15:09 2024-05-31 03:15:09 FUNCTION DEFINER utf8mb4 utf8mb4_0900_ai_ci utf8mb4_0900_ai_ci -2 concatenate 2 [{"name": "str1", "type": "varchar"}, {"name": "str2", "type": "varchar"}] varchar $1 + $2 sql udf_db test_account 2024-05-31 03:15:09 2024-05-31 03:15:09 FUNCTION DEFINER utf8mb4 utf8mb4_0900_ai_ci utf8mb4_0900_ai_ci +1 addab 2 [{"name": "x", "type": "int"}, {"name": "y", "type": "int"}] int $1 + $2 sql udf_db test_account 2024-08-07 09:45:39 2024-08-07 09:45:39 FUNCTION DEFINER utf8mb4 utf8mb4_0900_ai_ci utf8mb4_0900_ai_ci +2 concatenate 2 [{"name": "str1", "type": "varchar"}, {"name": "str2", "type": "varchar"}] varchar $1 + $2 sql udf_db test_account 2024-08-07 09:45:39 2024-08-07 09:45:39 FUNCTION DEFINER utf8mb4 utf8mb4_0900_ai_ci utf8mb4_0900_ai_ci drop database udf_db; drop snapshot udf_dsp01; drop snapshot udf_dsp02; @@ -87,7 +87,7 @@ language sql as '$1 + $2'; select * from mo_catalog.mo_user_defined_function; function_id name owner args rettype body language db definer modified_time created_time type security_type comment character_set_client collation_connection database_collation -4 addab 2 [{"name": "x", "type": "int"}, {"name": "y", "type": "int"}] int $1 + $2 sql udf_db2 test_account 2024-05-31 03:15:10 2024-05-31 03:15:10 FUNCTION DEFINER utf8mb4 utf8mb4_0900_ai_ci utf8mb4_0900_ai_ci +4 addab 2 [{"name": "x", "type": "int"}, {"name": "y", "type": "int"}] int $1 + $2 sql udf_db2 test_account 2024-08-07 09:45:40 2024-08-07 09:45:40 FUNCTION DEFINER utf8mb4 utf8mb4_0900_ai_ci utf8mb4_0900_ai_ci drop snapshot if exists udf_sp04; create snapshot udf_sp04 for account acc01; drop database udf_db2; @@ -96,7 +96,7 @@ function_id name owner args rettype body language db def restore account acc01 from snapshot udf_sp04 to account acc02; select * from mo_catalog.mo_user_defined_function; function_id name owner args rettype body language db definer modified_time created_time type security_type comment character_set_client collation_connection database_collation -4 addab 2 [{"name": "x", "type": "int"}, {"name": "y", "type": "int"}] int $1 + $2 sql udf_db2 test_account 2024-05-31 03:15:10 2024-05-31 03:15:10 FUNCTION DEFINER utf8mb4 utf8mb4_0900_ai_ci utf8mb4_0900_ai_ci +4 addab 2 [{"name": "x", "type": "int"}, {"name": "y", "type": "int"}] int $1 + $2 sql udf_db2 test_account 2024-08-07 09:45:40 2024-08-07 09:45:40 FUNCTION DEFINER utf8mb4 utf8mb4_0900_ai_ci utf8mb4_0900_ai_ci drop database udf_db2; drop snapshot udf_sp04; drop database if exists procedure_test; @@ -127,8 +127,8 @@ id val 3 c select * from mo_catalog.mo_stored_procedure; proc_id name creator args body db definer modified_time created_time type security_type comment character_set_client collation_connection database_collation -1 test_if_hit_elseif_first_elseif null {} begin DECLARE v1 INT; SET v1 = 5; IF v1 > 5 THEN select * from tbh1; ELSEIF v1 = 5 THEN select * from tbh2; ELSEIF v1 = 4 THEN select * from tbh2 limit 1; ELSE select * from tbh3; END IF; end procedure_test test_account 2024-05-31 03:15:11 2024-05-31 03:15:11 PROCEDURE DEFINER utf8mb4 utf8mb4_0900_ai_ci utf8mb4_0900_ai_ci -2 test_if_hit_if null {} begin DECLARE v1 INT; SET v1 = 5; IF v1 > 5 THEN select * from tbh1; ELSEIF v1 = 5 THEN select * from tbh2; ELSEIF v1 = 4 THEN select * from tbh2 limit 1; ELSE select * from tbh3; END IF; end procedure_test test_account 2024-05-31 03:15:11 2024-05-31 03:15:11 PROCEDURE DEFINER utf8mb4 utf8mb4_0900_ai_ci utf8mb4_0900_ai_ci +1 test_if_hit_elseif_first_elseif null {} begin DECLARE v1 INT; SET v1 = 5; IF v1 > 5 THEN select * from tbh1; ELSEIF v1 = 5 THEN select * from tbh2; ELSEIF v1 = 4 THEN select * from tbh2 limit 1; ELSE select * from tbh3; END IF; end procedure_test test_account 2024-08-07 09:45:40 2024-08-07 09:45:40 PROCEDURE DEFINER utf8mb4 utf8mb4_0900_ai_ci utf8mb4_0900_ai_ci +2 test_if_hit_if null {} begin DECLARE v1 INT; SET v1 = 5; IF v1 > 5 THEN select * from tbh1; ELSEIF v1 = 5 THEN select * from tbh2; ELSEIF v1 = 4 THEN select * from tbh2 limit 1; ELSE select * from tbh3; END IF; end procedure_test test_account 2024-08-07 09:45:40 2024-08-07 09:45:40 PROCEDURE DEFINER utf8mb4 utf8mb4_0900_ai_ci utf8mb4_0900_ai_ci drop snapshot if exists sp_sp05; create snapshot sp_sp05 for account acc01; drop procedure test_if_hit_elseif_first_elseif; @@ -136,8 +136,8 @@ drop procedure test_if_hit_if; restore account acc01 from snapshot sp_sp05 to account acc02; select * from mo_catalog.mo_stored_procedure; proc_id name creator args body db definer modified_time created_time type security_type comment character_set_client collation_connection database_collation -1 test_if_hit_elseif_first_elseif null {} begin DECLARE v1 INT; SET v1 = 5; IF v1 > 5 THEN select * from tbh1; ELSEIF v1 = 5 THEN select * from tbh2; ELSEIF v1 = 4 THEN select * from tbh2 limit 1; ELSE select * from tbh3; END IF; end procedure_test test_account 2024-05-31 03:15:11 2024-05-31 03:15:11 PROCEDURE DEFINER utf8mb4 utf8mb4_0900_ai_ci utf8mb4_0900_ai_ci -2 test_if_hit_if null {} begin DECLARE v1 INT; SET v1 = 5; IF v1 > 5 THEN select * from tbh1; ELSEIF v1 = 5 THEN select * from tbh2; ELSEIF v1 = 4 THEN select * from tbh2 limit 1; ELSE select * from tbh3; END IF; end procedure_test test_account 2024-05-31 03:15:11 2024-05-31 03:15:11 PROCEDURE DEFINER utf8mb4 utf8mb4_0900_ai_ci utf8mb4_0900_ai_ci +1 test_if_hit_elseif_first_elseif null {} begin DECLARE v1 INT; SET v1 = 5; IF v1 > 5 THEN select * from tbh1; ELSEIF v1 = 5 THEN select * from tbh2; ELSEIF v1 = 4 THEN select * from tbh2 limit 1; ELSE select * from tbh3; END IF; end procedure_test test_account 2024-08-07 09:45:40 2024-08-07 09:45:40 PROCEDURE DEFINER utf8mb4 utf8mb4_0900_ai_ci utf8mb4_0900_ai_ci +2 test_if_hit_if null {} begin DECLARE v1 INT; SET v1 = 5; IF v1 > 5 THEN select * from tbh1; ELSEIF v1 = 5 THEN select * from tbh2; ELSEIF v1 = 4 THEN select * from tbh2 limit 1; ELSE select * from tbh3; END IF; end procedure_test test_account 2024-08-07 09:45:40 2024-08-07 09:45:40 PROCEDURE DEFINER utf8mb4 utf8mb4_0900_ai_ci utf8mb4_0900_ai_ci call procedure_test.test_if_hit_elseif_first_elseif(); id val 1 a @@ -181,8 +181,8 @@ id val 3 3.5 select * from mo_catalog.mo_stored_procedure; proc_id name creator args body db definer modified_time created_time type security_type comment character_set_client collation_connection database_collation -3 test_if_hit_second_elseif null {} begin DECLARE v1 INT; SET v1 = 4; IF v1 > 5 THEN select * from tbh1; ELSEIF v1 = 5 THEN select * from tbh2; ELSEIF v1 = 4 THEN select * from tbh2 limit 1; ELSE select * from tbh3; END IF; end procedure_test test_account 2024-05-31 03:15:12 2024-05-31 03:15:12 PROCEDURE DEFINER utf8mb4 utf8mb4_0900_ai_ci utf8mb4_0900_ai_ci -4 test_if_hit_else null {} begin DECLARE v1 INT; SET v1 = 3; IF v1 > 5 THEN select * from tbh1; ELSEIF v1 = 5 THEN select * from tbh2; ELSEIF v1 = 4 THEN select * from tbh2 limit 1; ELSE select * from tbh3; END IF; end procedure_test test_account 2024-05-31 03:15:12 2024-05-31 03:15:12 PROCEDURE DEFINER utf8mb4 utf8mb4_0900_ai_ci utf8mb4_0900_ai_ci +3 test_if_hit_second_elseif null {} begin DECLARE v1 INT; SET v1 = 4; IF v1 > 5 THEN select * from tbh1; ELSEIF v1 = 5 THEN select * from tbh2; ELSEIF v1 = 4 THEN select * from tbh2 limit 1; ELSE select * from tbh3; END IF; end procedure_test test_account 2024-08-07 09:45:41 2024-08-07 09:45:41 PROCEDURE DEFINER utf8mb4 utf8mb4_0900_ai_ci utf8mb4_0900_ai_ci +4 test_if_hit_else null {} begin DECLARE v1 INT; SET v1 = 3; IF v1 > 5 THEN select * from tbh1; ELSEIF v1 = 5 THEN select * from tbh2; ELSEIF v1 = 4 THEN select * from tbh2 limit 1; ELSE select * from tbh3; END IF; end procedure_test test_account 2024-08-07 09:45:41 2024-08-07 09:45:41 PROCEDURE DEFINER utf8mb4 utf8mb4_0900_ai_ci utf8mb4_0900_ai_ci drop snapshot if exists sp_sp06; create snapshot sp_sp06 for account acc01; drop table tbh1; @@ -190,7 +190,7 @@ drop table tbh2; drop procedure test_if_hit_second_elseif; select * from mo_catalog.mo_stored_procedure; proc_id name creator args body db definer modified_time created_time type security_type comment character_set_client collation_connection database_collation -4 test_if_hit_else null {} begin DECLARE v1 INT; SET v1 = 3; IF v1 > 5 THEN select * from tbh1; ELSEIF v1 = 5 THEN select * from tbh2; ELSEIF v1 = 4 THEN select * from tbh2 limit 1; ELSE select * from tbh3; END IF; end procedure_test test_account 2024-05-31 03:15:12 2024-05-31 03:15:12 PROCEDURE DEFINER utf8mb4 utf8mb4_0900_ai_ci utf8mb4_0900_ai_ci +4 test_if_hit_else null {} begin DECLARE v1 INT; SET v1 = 3; IF v1 > 5 THEN select * from tbh1; ELSEIF v1 = 5 THEN select * from tbh2; ELSEIF v1 = 4 THEN select * from tbh2 limit 1; ELSE select * from tbh3; END IF; end procedure_test test_account 2024-08-07 09:45:41 2024-08-07 09:45:41 PROCEDURE DEFINER utf8mb4 utf8mb4_0900_ai_ci utf8mb4_0900_ai_ci restore account acc01 from snapshot sp_sp06 to account acc02; call procedure_test.test_if_hit_else(); id val @@ -202,8 +202,8 @@ id val 1 a select * from mo_catalog.mo_stored_procedure; proc_id name creator args body db definer modified_time created_time type security_type comment character_set_client collation_connection database_collation -3 test_if_hit_second_elseif null {} begin DECLARE v1 INT; SET v1 = 4; IF v1 > 5 THEN select * from tbh1; ELSEIF v1 = 5 THEN select * from tbh2; ELSEIF v1 = 4 THEN select * from tbh2 limit 1; ELSE select * from tbh3; END IF; end procedure_test test_account 2024-05-31 03:15:12 2024-05-31 03:15:12 PROCEDURE DEFINER utf8mb4 utf8mb4_0900_ai_ci utf8mb4_0900_ai_ci -4 test_if_hit_else null {} begin DECLARE v1 INT; SET v1 = 3; IF v1 > 5 THEN select * from tbh1; ELSEIF v1 = 5 THEN select * from tbh2; ELSEIF v1 = 4 THEN select * from tbh2 limit 1; ELSE select * from tbh3; END IF; end procedure_test test_account 2024-05-31 03:15:12 2024-05-31 03:15:12 PROCEDURE DEFINER utf8mb4 utf8mb4_0900_ai_ci utf8mb4_0900_ai_ci +3 test_if_hit_second_elseif null {} begin DECLARE v1 INT; SET v1 = 4; IF v1 > 5 THEN select * from tbh1; ELSEIF v1 = 5 THEN select * from tbh2; ELSEIF v1 = 4 THEN select * from tbh2 limit 1; ELSE select * from tbh3; END IF; end procedure_test test_account 2024-08-07 09:45:41 2024-08-07 09:45:41 PROCEDURE DEFINER utf8mb4 utf8mb4_0900_ai_ci utf8mb4_0900_ai_ci +4 test_if_hit_else null {} begin DECLARE v1 INT; SET v1 = 3; IF v1 > 5 THEN select * from tbh1; ELSEIF v1 = 5 THEN select * from tbh2; ELSEIF v1 = 4 THEN select * from tbh2 limit 1; ELSE select * from tbh3; END IF; end procedure_test test_account 2024-08-07 09:45:41 2024-08-07 09:45:41 PROCEDURE DEFINER utf8mb4 utf8mb4_0900_ai_ci utf8mb4_0900_ai_ci drop procedure procedure_test.test_if_hit_second_elseif; drop procedure procedure_test.test_if_hit_else; drop database procedure_test; @@ -214,8 +214,8 @@ drop stage if exists my_ext_stage1; create stage my_ext_stage1 URL='s3://load/files/' CREDENTIALS={'AWS_KEY_ID'='1a2b3c' ,'AWS_SECRET_KEY'='4x5y6z'}; select * from mo_catalog.mo_stages; stage_id stage_name url stage_credentials stage_status created_time comment -1 my_ext_stage s3://load/files/ disabled 2024-05-31 03:15:13 -2 my_ext_stage1 s3://load/files/ *0E0591E0E0FC0806815D04D93FD4BD3F5ADB39AD disabled 2024-05-31 03:15:13 +1 my_ext_stage s3://load/files/ disabled 2024-08-07 09:45:41 +2 my_ext_stage1 s3://load/files/ AWS_KEY_ID=1a2b3c,AWS_SECRET_KEY=4x5y6z disabled 2024-08-07 09:45:41 drop snapshot if exists stage_sp01; create snapshot stage_sp01 for account acc01; alter stage my_ext_stage1 SET URL='s3://load/files2/'; @@ -223,8 +223,8 @@ drop stage my_ext_stage; restore account acc01 from snapshot stage_sp01 to account acc02; select * from mo_catalog.mo_stages; stage_id stage_name url stage_credentials stage_status created_time comment -1 my_ext_stage s3://load/files/ disabled 2024-05-31 03:15:13 -2 my_ext_stage1 s3://load/files/ *0E0591E0E0FC0806815D04D93FD4BD3F5ADB39AD disabled 2024-05-31 03:15:13 +1 my_ext_stage s3://load/files/ disabled 2024-08-07 09:45:41 +2 my_ext_stage1 s3://load/files/ AWS_KEY_ID=1a2b3c,AWS_SECRET_KEY=4x5y6z disabled 2024-08-07 09:45:41 drop stage my_ext_stage; drop stage my_ext_stage1; drop snapshot stage_sp01; diff --git a/test/distributed/cases/snapshot/sys_restore_system_table_to_nonsys_account.result b/test/distributed/cases/snapshot/sys_restore_system_table_to_nonsys_account.result index 1ff63b327dc83..ad363acbd421d 100644 --- a/test/distributed/cases/snapshot/sys_restore_system_table_to_nonsys_account.result +++ b/test/distributed/cases/snapshot/sys_restore_system_table_to_nonsys_account.result @@ -19,7 +19,7 @@ drop snapshot if exists udf_dsp01; create snapshot udf_dsp01 for account acc01; show snapshots; SNAPSHOT_NAME TIMESTAMP SNAPSHOT_LEVEL ACCOUNT_NAME DATABASE_NAME TABLE_NAME -udf_dsp01 2024-05-31 03:17:51.505792 account acc01 +udf_dsp01 2024-08-07 09:46:00.17249539 account acc01 create function`concatenate`(str1 varchar(255), str2 varchar(255)) returns varchar(255) language sql as '$1 + $2'; @@ -30,8 +30,8 @@ drop snapshot if exists udf_dsp02; create snapshot udf_dsp02 for account acc01; show snapshots; SNAPSHOT_NAME TIMESTAMP SNAPSHOT_LEVEL ACCOUNT_NAME DATABASE_NAME TABLE_NAME -udf_dsp02 2024-05-31 03:17:51.605929 account acc01 -udf_dsp01 2024-05-31 03:17:51.505792 account acc01 +udf_dsp02 2024-08-07 09:46:00.188348149 account acc01 +udf_dsp01 2024-08-07 09:46:00.17249539 account acc01 drop database if exists udf_db2; create database udf_db2; use udf_db2; @@ -50,27 +50,27 @@ drop snapshot if exists udf_dsp03; create snapshot udf_dsp03 for account acc01; show snapshots; SNAPSHOT_NAME TIMESTAMP SNAPSHOT_LEVEL ACCOUNT_NAME DATABASE_NAME TABLE_NAME -udf_dsp03 2024-05-31 03:17:51.743983 account acc01 -udf_dsp02 2024-05-31 03:17:51.605929 account acc01 -udf_dsp01 2024-05-31 03:17:51.505792 account acc01 +udf_dsp03 2024-08-07 09:46:00.221115771 account acc01 +udf_dsp02 2024-08-07 09:46:00.188348149 account acc01 +udf_dsp01 2024-08-07 09:46:00.17249539 account acc01 select * from mo_catalog.mo_user_defined_function; function_id name owner args rettype body language db definer modified_time created_time type security_type comment character_set_client collation_connection database_collation -1 addab 2 [{"name": "x", "type": "int"}, {"name": "y", "type": "int"}] int $1 + $2 sql udf_db test_account 2024-05-31 03:17:51 2024-05-31 03:17:51 FUNCTION DEFINER utf8mb4 utf8mb4_0900_ai_ci utf8mb4_0900_ai_ci -2 concatenate 2 [{"name": "str1", "type": "varchar"}, {"name": "str2", "type": "varchar"}] varchar $1 + $2 sql udf_db test_account 2024-05-31 03:17:51 2024-05-31 03:17:51 FUNCTION DEFINER utf8mb4 utf8mb4_0900_ai_ci utf8mb4_0900_ai_ci -3 subab 2 [{"name": "x", "type": "int"}, {"name": "y", "type": "int"}] int $1 - $2 sql udf_db2 test_account 2024-05-31 03:17:51 2024-05-31 03:17:51 FUNCTION DEFINER utf8mb4 utf8mb4_0900_ai_ci utf8mb4_0900_ai_ci +1 addab 2 [{"name": "x", "type": "int"}, {"name": "y", "type": "int"}] int $1 + $2 sql udf_db test_account 2024-08-07 09:46:00 2024-08-07 09:46:00 FUNCTION DEFINER utf8mb4 utf8mb4_0900_ai_ci utf8mb4_0900_ai_ci +2 concatenate 2 [{"name": "str1", "type": "varchar"}, {"name": "str2", "type": "varchar"}] varchar $1 + $2 sql udf_db test_account 2024-08-07 09:46:00 2024-08-07 09:46:00 FUNCTION DEFINER utf8mb4 utf8mb4_0900_ai_ci utf8mb4_0900_ai_ci +3 subab 2 [{"name": "x", "type": "int"}, {"name": "y", "type": "int"}] int $1 - $2 sql udf_db2 test_account 2024-08-07 09:46:00 2024-08-07 09:46:00 FUNCTION DEFINER utf8mb4 utf8mb4_0900_ai_ci utf8mb4_0900_ai_ci drop function subab(x int,y int); drop function udf_db.concatenate(str1 varchar(255), str2 varchar(255)); restore account acc01 from snapshot udf_dsp03; select * from mo_catalog.mo_user_defined_function; function_id name owner args rettype body language db definer modified_time created_time type security_type comment character_set_client collation_connection database_collation -1 addab 2 [{"name": "x", "type": "int"}, {"name": "y", "type": "int"}] int $1 + $2 sql udf_db test_account 2024-05-31 03:17:51 2024-05-31 03:17:51 FUNCTION DEFINER utf8mb4 utf8mb4_0900_ai_ci utf8mb4_0900_ai_ci -2 concatenate 2 [{"name": "str1", "type": "varchar"}, {"name": "str2", "type": "varchar"}] varchar $1 + $2 sql udf_db test_account 2024-05-31 03:17:51 2024-05-31 03:17:51 FUNCTION DEFINER utf8mb4 utf8mb4_0900_ai_ci utf8mb4_0900_ai_ci -3 subab 2 [{"name": "x", "type": "int"}, {"name": "y", "type": "int"}] int $1 - $2 sql udf_db2 test_account 2024-05-31 03:17:51 2024-05-31 03:17:51 FUNCTION DEFINER utf8mb4 utf8mb4_0900_ai_ci utf8mb4_0900_ai_ci +1 addab 2 [{"name": "x", "type": "int"}, {"name": "y", "type": "int"}] int $1 + $2 sql udf_db test_account 2024-08-07 09:46:00 2024-08-07 09:46:00 FUNCTION DEFINER utf8mb4 utf8mb4_0900_ai_ci utf8mb4_0900_ai_ci +2 concatenate 2 [{"name": "str1", "type": "varchar"}, {"name": "str2", "type": "varchar"}] varchar $1 + $2 sql udf_db test_account 2024-08-07 09:46:00 2024-08-07 09:46:00 FUNCTION DEFINER utf8mb4 utf8mb4_0900_ai_ci utf8mb4_0900_ai_ci +3 subab 2 [{"name": "x", "type": "int"}, {"name": "y", "type": "int"}] int $1 - $2 sql udf_db2 test_account 2024-08-07 09:46:00 2024-08-07 09:46:00 FUNCTION DEFINER utf8mb4 utf8mb4_0900_ai_ci utf8mb4_0900_ai_ci restore account acc01 from snapshot udf_dsp02; select * from mo_catalog.mo_user_defined_function; function_id name owner args rettype body language db definer modified_time created_time type security_type comment character_set_client collation_connection database_collation -1 addab 2 [{"name": "x", "type": "int"}, {"name": "y", "type": "int"}] int $1 + $2 sql udf_db test_account 2024-05-31 03:17:51 2024-05-31 03:17:51 FUNCTION DEFINER utf8mb4 utf8mb4_0900_ai_ci utf8mb4_0900_ai_ci -2 concatenate 2 [{"name": "str1", "type": "varchar"}, {"name": "str2", "type": "varchar"}] varchar $1 + $2 sql udf_db test_account 2024-05-31 03:17:51 2024-05-31 03:17:51 FUNCTION DEFINER utf8mb4 utf8mb4_0900_ai_ci utf8mb4_0900_ai_ci +1 addab 2 [{"name": "x", "type": "int"}, {"name": "y", "type": "int"}] int $1 + $2 sql udf_db test_account 2024-08-07 09:46:00 2024-08-07 09:46:00 FUNCTION DEFINER utf8mb4 utf8mb4_0900_ai_ci utf8mb4_0900_ai_ci +2 concatenate 2 [{"name": "str1", "type": "varchar"}, {"name": "str2", "type": "varchar"}] varchar $1 + $2 sql udf_db test_account 2024-08-07 09:46:00 2024-08-07 09:46:00 FUNCTION DEFINER utf8mb4 utf8mb4_0900_ai_ci utf8mb4_0900_ai_ci drop database udf_db; drop snapshot udf_dsp01; drop snapshot udf_dsp02; @@ -83,7 +83,7 @@ language sql as '$1 + $2'; select * from mo_catalog.mo_user_defined_function; function_id name owner args rettype body language db definer modified_time created_time type security_type comment character_set_client collation_connection database_collation -3 addab 2 [{"name": "x", "type": "int"}, {"name": "y", "type": "int"}] int $1 + $2 sql udf_db2 test_account 2024-05-31 03:17:52 2024-05-31 03:17:52 FUNCTION DEFINER utf8mb4 utf8mb4_0900_ai_ci utf8mb4_0900_ai_ci +3 addab 2 [{"name": "x", "type": "int"}, {"name": "y", "type": "int"}] int $1 + $2 sql udf_db2 test_account 2024-08-07 09:46:00 2024-08-07 09:46:00 FUNCTION DEFINER utf8mb4 utf8mb4_0900_ai_ci utf8mb4_0900_ai_ci drop snapshot if exists udf_sp04; create snapshot udf_sp04 for account acc01; drop database udf_db2; @@ -92,7 +92,7 @@ function_id name owner args rettype body language db def restore account acc01 from snapshot udf_sp04; select * from mo_catalog.mo_user_defined_function; function_id name owner args rettype body language db definer modified_time created_time type security_type comment character_set_client collation_connection database_collation -3 addab 2 [{"name": "x", "type": "int"}, {"name": "y", "type": "int"}] int $1 + $2 sql udf_db2 test_account 2024-05-31 03:17:52 2024-05-31 03:17:52 FUNCTION DEFINER utf8mb4 utf8mb4_0900_ai_ci utf8mb4_0900_ai_ci +3 addab 2 [{"name": "x", "type": "int"}, {"name": "y", "type": "int"}] int $1 + $2 sql udf_db2 test_account 2024-08-07 09:46:00 2024-08-07 09:46:00 FUNCTION DEFINER utf8mb4 utf8mb4_0900_ai_ci utf8mb4_0900_ai_ci drop database udf_db2; drop snapshot udf_sp04; drop database if exists procedure_test; @@ -123,8 +123,8 @@ id val 3 c select * from mo_catalog.mo_stored_procedure; proc_id name creator args body db definer modified_time created_time type security_type comment character_set_client collation_connection database_collation -1 test_if_hit_elseif_first_elseif null {} begin DECLARE v1 INT; SET v1 = 5; IF v1 > 5 THEN select * from tbh1; ELSEIF v1 = 5 THEN select * from tbh2; ELSEIF v1 = 4 THEN select * from tbh2 limit 1; ELSE select * from tbh3; END IF; end procedure_test test_account 2024-05-31 03:17:53 2024-05-31 03:17:53 PROCEDURE DEFINER utf8mb4 utf8mb4_0900_ai_ci utf8mb4_0900_ai_ci -2 test_if_hit_if null {} begin DECLARE v1 INT; SET v1 = 5; IF v1 > 5 THEN select * from tbh1; ELSEIF v1 = 5 THEN select * from tbh2; ELSEIF v1 = 4 THEN select * from tbh2 limit 1; ELSE select * from tbh3; END IF; end procedure_test test_account 2024-05-31 03:17:53 2024-05-31 03:17:53 PROCEDURE DEFINER utf8mb4 utf8mb4_0900_ai_ci utf8mb4_0900_ai_ci +1 test_if_hit_elseif_first_elseif null {} begin DECLARE v1 INT; SET v1 = 5; IF v1 > 5 THEN select * from tbh1; ELSEIF v1 = 5 THEN select * from tbh2; ELSEIF v1 = 4 THEN select * from tbh2 limit 1; ELSE select * from tbh3; END IF; end procedure_test test_account 2024-08-07 09:46:00 2024-08-07 09:46:00 PROCEDURE DEFINER utf8mb4 utf8mb4_0900_ai_ci utf8mb4_0900_ai_ci +2 test_if_hit_if null {} begin DECLARE v1 INT; SET v1 = 5; IF v1 > 5 THEN select * from tbh1; ELSEIF v1 = 5 THEN select * from tbh2; ELSEIF v1 = 4 THEN select * from tbh2 limit 1; ELSE select * from tbh3; END IF; end procedure_test test_account 2024-08-07 09:46:00 2024-08-07 09:46:00 PROCEDURE DEFINER utf8mb4 utf8mb4_0900_ai_ci utf8mb4_0900_ai_ci drop snapshot if exists sp_sp05; create snapshot sp_sp05 for account acc01; drop procedure test_if_hit_elseif_first_elseif; @@ -132,8 +132,8 @@ drop procedure test_if_hit_if; restore account acc01 from snapshot sp_sp05; select * from mo_catalog.mo_stored_procedure; proc_id name creator args body db definer modified_time created_time type security_type comment character_set_client collation_connection database_collation -1 test_if_hit_elseif_first_elseif null {} begin DECLARE v1 INT; SET v1 = 5; IF v1 > 5 THEN select * from tbh1; ELSEIF v1 = 5 THEN select * from tbh2; ELSEIF v1 = 4 THEN select * from tbh2 limit 1; ELSE select * from tbh3; END IF; end procedure_test test_account 2024-05-31 03:17:53 2024-05-31 03:17:53 PROCEDURE DEFINER utf8mb4 utf8mb4_0900_ai_ci utf8mb4_0900_ai_ci -2 test_if_hit_if null {} begin DECLARE v1 INT; SET v1 = 5; IF v1 > 5 THEN select * from tbh1; ELSEIF v1 = 5 THEN select * from tbh2; ELSEIF v1 = 4 THEN select * from tbh2 limit 1; ELSE select * from tbh3; END IF; end procedure_test test_account 2024-05-31 03:17:53 2024-05-31 03:17:53 PROCEDURE DEFINER utf8mb4 utf8mb4_0900_ai_ci utf8mb4_0900_ai_ci +1 test_if_hit_elseif_first_elseif null {} begin DECLARE v1 INT; SET v1 = 5; IF v1 > 5 THEN select * from tbh1; ELSEIF v1 = 5 THEN select * from tbh2; ELSEIF v1 = 4 THEN select * from tbh2 limit 1; ELSE select * from tbh3; END IF; end procedure_test test_account 2024-08-07 09:46:00 2024-08-07 09:46:00 PROCEDURE DEFINER utf8mb4 utf8mb4_0900_ai_ci utf8mb4_0900_ai_ci +2 test_if_hit_if null {} begin DECLARE v1 INT; SET v1 = 5; IF v1 > 5 THEN select * from tbh1; ELSEIF v1 = 5 THEN select * from tbh2; ELSEIF v1 = 4 THEN select * from tbh2 limit 1; ELSE select * from tbh3; END IF; end procedure_test test_account 2024-08-07 09:46:00 2024-08-07 09:46:00 PROCEDURE DEFINER utf8mb4 utf8mb4_0900_ai_ci utf8mb4_0900_ai_ci call test_if_hit_elseif_first_elseif(); id val 1 a @@ -174,8 +174,8 @@ id val 3 3.5 select * from mo_catalog.mo_stored_procedure; proc_id name creator args body db definer modified_time created_time type security_type comment character_set_client collation_connection database_collation -3 test_if_hit_second_elseif null {} begin DECLARE v1 INT; SET v1 = 4; IF v1 > 5 THEN select * from tbh1; ELSEIF v1 = 5 THEN select * from tbh2; ELSEIF v1 = 4 THEN select * from tbh2 limit 1; ELSE select * from tbh3; END IF; end procedure_test test_account 2024-05-31 03:17:54 2024-05-31 03:17:54 PROCEDURE DEFINER utf8mb4 utf8mb4_0900_ai_ci utf8mb4_0900_ai_ci -4 test_if_hit_else null {} begin DECLARE v1 INT; SET v1 = 3; IF v1 > 5 THEN select * from tbh1; ELSEIF v1 = 5 THEN select * from tbh2; ELSEIF v1 = 4 THEN select * from tbh2 limit 1; ELSE select * from tbh3; END IF; end procedure_test test_account 2024-05-31 03:17:54 2024-05-31 03:17:54 PROCEDURE DEFINER utf8mb4 utf8mb4_0900_ai_ci utf8mb4_0900_ai_ci +3 test_if_hit_second_elseif null {} begin DECLARE v1 INT; SET v1 = 4; IF v1 > 5 THEN select * from tbh1; ELSEIF v1 = 5 THEN select * from tbh2; ELSEIF v1 = 4 THEN select * from tbh2 limit 1; ELSE select * from tbh3; END IF; end procedure_test test_account 2024-08-07 09:46:01 2024-08-07 09:46:01 PROCEDURE DEFINER utf8mb4 utf8mb4_0900_ai_ci utf8mb4_0900_ai_ci +4 test_if_hit_else null {} begin DECLARE v1 INT; SET v1 = 3; IF v1 > 5 THEN select * from tbh1; ELSEIF v1 = 5 THEN select * from tbh2; ELSEIF v1 = 4 THEN select * from tbh2 limit 1; ELSE select * from tbh3; END IF; end procedure_test test_account 2024-08-07 09:46:01 2024-08-07 09:46:01 PROCEDURE DEFINER utf8mb4 utf8mb4_0900_ai_ci utf8mb4_0900_ai_ci drop snapshot if exists sp_sp06; create snapshot sp_sp06 for account acc01; drop table tbh1; @@ -183,7 +183,7 @@ drop table tbh2; drop procedure test_if_hit_second_elseif; select * from mo_catalog.mo_stored_procedure; proc_id name creator args body db definer modified_time created_time type security_type comment character_set_client collation_connection database_collation -4 test_if_hit_else null {} begin DECLARE v1 INT; SET v1 = 3; IF v1 > 5 THEN select * from tbh1; ELSEIF v1 = 5 THEN select * from tbh2; ELSEIF v1 = 4 THEN select * from tbh2 limit 1; ELSE select * from tbh3; END IF; end procedure_test test_account 2024-05-31 03:17:54 2024-05-31 03:17:54 PROCEDURE DEFINER utf8mb4 utf8mb4_0900_ai_ci utf8mb4_0900_ai_ci +4 test_if_hit_else null {} begin DECLARE v1 INT; SET v1 = 3; IF v1 > 5 THEN select * from tbh1; ELSEIF v1 = 5 THEN select * from tbh2; ELSEIF v1 = 4 THEN select * from tbh2 limit 1; ELSE select * from tbh3; END IF; end procedure_test test_account 2024-08-07 09:46:01 2024-08-07 09:46:01 PROCEDURE DEFINER utf8mb4 utf8mb4_0900_ai_ci utf8mb4_0900_ai_ci restore account acc01 from snapshot sp_sp06; call test_if_hit_else(); id val @@ -195,8 +195,8 @@ id val 1 a select * from mo_catalog.mo_stored_procedure; proc_id name creator args body db definer modified_time created_time type security_type comment character_set_client collation_connection database_collation -3 test_if_hit_second_elseif null {} begin DECLARE v1 INT; SET v1 = 4; IF v1 > 5 THEN select * from tbh1; ELSEIF v1 = 5 THEN select * from tbh2; ELSEIF v1 = 4 THEN select * from tbh2 limit 1; ELSE select * from tbh3; END IF; end procedure_test test_account 2024-05-31 03:17:54 2024-05-31 03:17:54 PROCEDURE DEFINER utf8mb4 utf8mb4_0900_ai_ci utf8mb4_0900_ai_ci -4 test_if_hit_else null {} begin DECLARE v1 INT; SET v1 = 3; IF v1 > 5 THEN select * from tbh1; ELSEIF v1 = 5 THEN select * from tbh2; ELSEIF v1 = 4 THEN select * from tbh2 limit 1; ELSE select * from tbh3; END IF; end procedure_test test_account 2024-05-31 03:17:54 2024-05-31 03:17:54 PROCEDURE DEFINER utf8mb4 utf8mb4_0900_ai_ci utf8mb4_0900_ai_ci +3 test_if_hit_second_elseif null {} begin DECLARE v1 INT; SET v1 = 4; IF v1 > 5 THEN select * from tbh1; ELSEIF v1 = 5 THEN select * from tbh2; ELSEIF v1 = 4 THEN select * from tbh2 limit 1; ELSE select * from tbh3; END IF; end procedure_test test_account 2024-08-07 09:46:01 2024-08-07 09:46:01 PROCEDURE DEFINER utf8mb4 utf8mb4_0900_ai_ci utf8mb4_0900_ai_ci +4 test_if_hit_else null {} begin DECLARE v1 INT; SET v1 = 3; IF v1 > 5 THEN select * from tbh1; ELSEIF v1 = 5 THEN select * from tbh2; ELSEIF v1 = 4 THEN select * from tbh2 limit 1; ELSE select * from tbh3; END IF; end procedure_test test_account 2024-08-07 09:46:01 2024-08-07 09:46:01 PROCEDURE DEFINER utf8mb4 utf8mb4_0900_ai_ci utf8mb4_0900_ai_ci drop procedure test_if_hit_second_elseif; drop procedure test_if_hit_else; drop snapshot sp_sp06; @@ -206,20 +206,20 @@ drop stage if exists my_ext_stage1; create stage my_ext_stage1 URL='s3://load/files/' CREDENTIALS={'AWS_KEY_ID'='1a2b3c' ,'AWS_SECRET_KEY'='4x5y6z'}; select * from mo_catalog.mo_stages; stage_id stage_name url stage_credentials stage_status created_time comment -1 my_ext_stage s3://load/files/ disabled 2024-05-31 03:17:55 -2 my_ext_stage1 s3://load/files/ *0E0591E0E0FC0806815D04D93FD4BD3F5ADB39AD disabled 2024-05-31 03:17:55 +1 my_ext_stage s3://load/files/ disabled 2024-08-07 09:46:01 +2 my_ext_stage1 s3://load/files/ AWS_KEY_ID=1a2b3c,AWS_SECRET_KEY=4x5y6z disabled 2024-08-07 09:46:01 drop snapshot if exists stage_sp01; create snapshot stage_sp01 for account acc01; alter stage my_ext_stage1 SET URL='s3://load/files2/'; drop stage my_ext_stage; select * from mo_catalog.mo_stages; stage_id stage_name url stage_credentials stage_status created_time comment -2 my_ext_stage1 s3://load/files2/ *0E0591E0E0FC0806815D04D93FD4BD3F5ADB39AD disabled 2024-05-31 03:17:55 +2 my_ext_stage1 s3://load/files2/ AWS_KEY_ID=1a2b3c,AWS_SECRET_KEY=4x5y6z disabled 2024-08-07 09:46:01 restore account acc01 from snapshot stage_sp01; select * from mo_catalog.mo_stages; stage_id stage_name url stage_credentials stage_status created_time comment -1 my_ext_stage s3://load/files/ disabled 2024-05-31 03:17:55 -2 my_ext_stage1 s3://load/files/ *0E0591E0E0FC0806815D04D93FD4BD3F5ADB39AD disabled 2024-05-31 03:17:55 +1 my_ext_stage s3://load/files/ disabled 2024-08-07 09:46:01 +2 my_ext_stage1 s3://load/files/ AWS_KEY_ID=1a2b3c,AWS_SECRET_KEY=4x5y6z disabled 2024-08-07 09:46:01 drop stage my_ext_stage; drop stage my_ext_stage1; drop snapshot stage_sp01; diff --git a/test/distributed/cases/stage/stage.result b/test/distributed/cases/stage/stage.result index c699a5108857d..eaa24aa35aae1 100644 --- a/test/distributed/cases/stage/stage.result +++ b/test/distributed/cases/stage/stage.result @@ -1,30 +1,40 @@ -CREATE STAGE my_ext_stage URL='s3://load/files/'; -CREATE STAGE my_ext_stage URL='s3://load/files/'; +CREATE STAGE invalid_path_stage URL='/path/to'; +invalid configuration: URL protocol only supports stage://, s3:// and file:/// +CREATE STAGE invalid_unknown_protocol_stage URL='minio:///path/to'; +invalid configuration: URL protocol only supports stage://, s3:// and file:/// +CREATE STAGE my_ext_stage URL='s3://bucket/files/'; +CREATE STAGE my_sub_stage URL='stage://my_ext_stage/a/b/c/'; +CREATE STAGE my_ext_stage URL='s3://bucket/files/'; internal error: the stage my_ext_stage exists -CREATE STAGE if not exists my_ext_stage URL='s3://load/files/'; -SELECT stage_name from mo_catalog.mo_stages; -stage_name -my_ext_stage -CREATE STAGE my_ext_stage1 URL='s3://load/files/' CREDENTIALS={'AWS_KEY_ID'='1a2b3c' ,'AWS_SECRET_KEY'='4x5y6z'}; -CREATE STAGE my_ext_stage2 URL='s3://load/files/' CREDENTIALS={'AWS_KEY_ID'='1a2b3c' ,'AWS_SECRET_KEY'='4x5y6z'}; -SELECT stage_name, stage_status from mo_catalog.mo_stages; -stage_name stage_status -my_ext_stage disabled -my_ext_stage1 disabled -my_ext_stage2 disabled -CREATE STAGE my_ext_stage3 URL='s3://load/files/' CREDENTIALS={'AWS_KEY_ID'='1a2b3c' ,'AWS_SECRET_KEY'='4x5y6z'} ENABLE = TRUE; -SELECT stage_name, stage_status from mo_catalog.mo_stages; -stage_name stage_status -my_ext_stage disabled -my_ext_stage1 disabled -my_ext_stage2 disabled -my_ext_stage3 enabled -ALTER STAGE my_ext_stage4 SET URL='s3://load/files2/'; +ALTER STAGE my_ext_stage SET URL='abc'; +invalid configuration: URL protocol only supports stage://, s3:// and file:/// +CREATE STAGE if not exists my_ext_stage URL='s3://bucket/files/'; +SELECT * from mo_catalog.mo_stages; +stage_id stage_name url stage_credentials stage_status created_time comment +22 my_ext_stage s3://bucket/files/ disabled 2024-08-21 10:45:44 +23 my_sub_stage stage://my_ext_stage/a/b/c/ disabled 2024-08-21 10:45:44 +CREATE STAGE my_ext_stage1 URL='s3://bucket1/files/' CREDENTIALS={'AWS_KEY_ID'='1a2b3c' ,'AWS_SECRET_KEY'='4x5y6z', 'AWS_REGION'='us-east-2', 'PROVIDER'='minio'}; +CREATE STAGE my_ext_stage2 URL='s3://bucket2/files/' CREDENTIALS={'AWS_KEY_ID'='1a2b3c' ,'AWS_SECRET_KEY'='4x5y6z', 'AWS_REGION'='us-east-2', 'PROVIDER'='minio'}; +SELECT * from mo_catalog.mo_stages; +stage_id stage_name url stage_credentials stage_status created_time comment +22 my_ext_stage s3://bucket/files/ disabled 2024-08-21 10:45:44 +23 my_sub_stage stage://my_ext_stage/a/b/c/ disabled 2024-08-21 10:45:44 +24 my_ext_stage1 s3://bucket1/files/ AWS_KEY_ID=1a2b3c,AWS_SECRET_KEY=4x5y6z,AWS_REGION=us-east-2,PROVIDER=minio disabled 2024-08-21 10:45:44 +25 my_ext_stage2 s3://bucket2/files/ AWS_KEY_ID=1a2b3c,AWS_SECRET_KEY=4x5y6z,AWS_REGION=us-east-2,PROVIDER=minio disabled 2024-08-21 10:45:44 +CREATE STAGE my_ext_stage3 URL='s3://bucket3/files/' CREDENTIALS={'AWS_KEY_ID'='1a2b3c' ,'AWS_SECRET_KEY'='4x5y6z', 'AWS_REGION'='us-east-2', 'PROVIDER'='minio'}; +SELECT * from mo_catalog.mo_stages; +stage_id stage_name url stage_credentials stage_status created_time comment +22 my_ext_stage s3://bucket/files/ disabled 2024-08-21 10:45:44 +23 my_sub_stage stage://my_ext_stage/a/b/c/ disabled 2024-08-21 10:45:44 +24 my_ext_stage1 s3://bucket1/files/ AWS_KEY_ID=1a2b3c,AWS_SECRET_KEY=4x5y6z,AWS_REGION=us-east-2,PROVIDER=minio disabled 2024-08-21 10:45:44 +25 my_ext_stage2 s3://bucket2/files/ AWS_KEY_ID=1a2b3c,AWS_SECRET_KEY=4x5y6z,AWS_REGION=us-east-2,PROVIDER=minio disabled 2024-08-21 10:45:44 +26 my_ext_stage3 s3://bucket3/files/ AWS_KEY_ID=1a2b3c,AWS_SECRET_KEY=4x5y6z,AWS_REGION=us-east-2,PROVIDER=minio disabled 2024-08-21 10:45:44 +ALTER STAGE my_ext_stage4 SET URL='s3://bucket4/files2/'; internal error: the stage my_ext_stage4 not exists -ALTER STAGE if exists my_ext_stage4 SET URL='s3://load/files2/'; -ALTER STAGE my_ext_stage1 SET URL='s3://load/files2/' CREDENTIALS={'AWS_KEY_ID'='1a2b3d' ,'AWS_SECRET_KEY'='4x5y6z'}; +ALTER STAGE if exists my_ext_stage4 SET URL='s3://bucket4/files2/'; +ALTER STAGE my_ext_stage1 SET URL='s3://bucket2/files2/' CREDENTIALS={'AWS_KEY_ID'='1a2b3d' ,'AWS_SECRET_KEY'='4x5y6z'}; internal error: at most one option at a time -ALTER STAGE my_ext_stage1 SET URL='s3://load/files2/'; +ALTER STAGE my_ext_stage1 SET URL='s3://bucket2/files2/'; ALTER STAGE my_ext_stage1 SET CREDENTIALS={'AWS_KEY_ID'='1a2b3d' ,'AWS_SECRET_KEY'='4x5y6z'}; DROP STAGE my_ext_stage5; internal error: the stage my_ext_stage5 not exists @@ -33,47 +43,48 @@ DROP STAGE my_ext_stage; DROP STAGE my_ext_stage1; DROP STAGE my_ext_stage2; DROP STAGE my_ext_stage3; -CREATE STAGE my_ext_stage URL='s3://load/files/'; -SELECT stage_name from mo_catalog.mo_stages; -stage_name -my_ext_stage +DROP STAGE my_sub_stage; +CREATE STAGE my_ext_stage URL='s3://bucket/files/'; +SELECT * from mo_catalog.mo_stages; +stage_id stage_name url stage_credentials stage_status created_time comment +27 my_ext_stage s3://bucket/files/ disabled 2024-08-21 10:45:44 create account default_1 ADMIN_NAME admin IDENTIFIED BY '111111'; -CREATE STAGE my_ext_stage1 URL='s3://load/files/' CREDENTIALS={'AWS_KEY_ID'='1a2b3c' ,'AWS_SECRET_KEY'='4x5y6z'}; -CREATE STAGE my_ext_stage2 URL='s3://load/files/' CREDENTIALS={'AWS_KEY_ID'='1a2b3c' ,'AWS_SECRET_KEY'='4x5y6z'}; -SELECT stage_name, stage_status from mo_catalog.mo_stages; -stage_name stage_status -my_ext_stage1 disabled -my_ext_stage2 disabled +CREATE STAGE my_ext_stage1 URL='s3://bucket1/files/' CREDENTIALS={'AWS_KEY_ID'='1a2b3c' ,'AWS_SECRET_KEY'='4x5y6z', 'AWS_REGION'='us-east-2', 'PROVIDER'='minio'}; +CREATE STAGE my_ext_stage2 URL='s3://bucket2/files/' CREDENTIALS={'AWS_KEY_ID'='1a2b3c' ,'AWS_SECRET_KEY'='4x5y6z', 'AWS_REGION'='us-east-2', 'PROVIDER'='minio'}; +SELECT * from mo_catalog.mo_stages; +stage_id stage_name url stage_credentials stage_status created_time comment +1 my_ext_stage1 s3://bucket1/files/ AWS_KEY_ID=1a2b3c,AWS_SECRET_KEY=4x5y6z,AWS_REGION=us-east-2,PROVIDER=minio disabled 2024-08-21 10:45:45 +2 my_ext_stage2 s3://bucket2/files/ AWS_KEY_ID=1a2b3c,AWS_SECRET_KEY=4x5y6z,AWS_REGION=us-east-2,PROVIDER=minio disabled 2024-08-21 10:45:45 DROP STAGE my_ext_stage1; DROP STAGE my_ext_stage2; drop account default_1; drop stage my_ext_stage; -CREATE STAGE my_ext_stage URL='s3://load/files/'; -CREATE STAGE my_ext_stage URL='s3://load/files/'; +CREATE STAGE my_ext_stage URL='s3://bucket/files/'; +CREATE STAGE my_ext_stage URL='s3://bucket/files/'; internal error: the stage my_ext_stage exists -CREATE STAGE if not exists my_ext_stage URL='s3://load/files/'; +CREATE STAGE if not exists my_ext_stage URL='s3://bucket/files/'; SHOW STAGES; STAGE_NAME URL STATUS COMMENT -my_ext_stage s3://load/files/ DISABLED -CREATE STAGE my_ext_stage1 URL='s3://load/files/' CREDENTIALS={'AWS_KEY_ID'='1a2b3c' ,'AWS_SECRET_KEY'='4x5y6z'}; -CREATE STAGE my_ext_stage2 URL='s3://load/files/' CREDENTIALS={'AWS_KEY_ID'='1a2b3c' ,'AWS_SECRET_KEY'='4x5y6z'}; +my_ext_stage s3://bucket/files/ DISABLED +CREATE STAGE my_ext_stage1 URL='s3://bucket/files/' CREDENTIALS={'AWS_KEY_ID'='1a2b3c' ,'AWS_SECRET_KEY'='4x5y6z'}; +CREATE STAGE my_ext_stage2 URL='s3://bucket/files/' CREDENTIALS={'AWS_KEY_ID'='1a2b3c' ,'AWS_SECRET_KEY'='4x5y6z'}; SHOW STAGES; STAGE_NAME URL STATUS COMMENT -my_ext_stage s3://load/files/ DISABLED -my_ext_stage1 s3://load/files/ DISABLED -my_ext_stage2 s3://load/files/ DISABLED -CREATE STAGE my_ext_stage3 URL='s3://load/files/' CREDENTIALS={'AWS_KEY_ID'='1a2b3c' ,'AWS_SECRET_KEY'='4x5y6z'} ENABLE = TRUE; -CREATE STAGE my_ext_stage4 URL='s3://load/files/' CREDENTIALS={'AWS_KEY_ID'='1a2b3c' ,'AWS_SECRET_KEY'='4x5y6z'} ENABLE = TRUE COMMENT = 'self stage'; +my_ext_stage s3://bucket/files/ DISABLED +my_ext_stage1 s3://bucket/files/ DISABLED +my_ext_stage2 s3://bucket/files/ DISABLED +CREATE STAGE my_ext_stage3 URL='s3://bucket/files/' CREDENTIALS={'AWS_KEY_ID'='1a2b3c' ,'AWS_SECRET_KEY'='4x5y6z'}; +CREATE STAGE my_ext_stage4 URL='s3://bucket/files/' CREDENTIALS={'AWS_KEY_ID'='1a2b3c' ,'AWS_SECRET_KEY'='4x5y6z'} COMMENT = 'self stage'; SHOW STAGES; STAGE_NAME URL STATUS COMMENT -my_ext_stage s3://load/files/ DISABLED -my_ext_stage1 s3://load/files/ DISABLED -my_ext_stage2 s3://load/files/ DISABLED -my_ext_stage3 s3://load/files/ ENABLED -my_ext_stage4 s3://load/files/ ENABLED self stage +my_ext_stage s3://bucket/files/ DISABLED +my_ext_stage1 s3://bucket/files/ DISABLED +my_ext_stage2 s3://bucket/files/ DISABLED +my_ext_stage3 s3://bucket/files/ DISABLED +my_ext_stage4 s3://bucket/files/ DISABLED self stage SHOW STAGES like 'my_ext_stage3'; STAGE_NAME URL STATUS COMMENT -my_ext_stage3 s3://load/files/ ENABLED +my_ext_stage3 s3://bucket/files/ DISABLED ALTER STAGE my_ext_stage5 SET URL='s3://load/files2/'; internal error: the stage my_ext_stage5 not exists ALTER STAGE if exists my_ext_stage5 SET URL='s3://load/files2/'; @@ -84,14 +95,14 @@ ALTER STAGE my_ext_stage1 SET CREDENTIALS={'AWS_KEY_ID'='1a2b3d' ,'AWS_SECRET_KE ALTER STAGE my_ext_stage4 SET COMMENT = 'user stage'; SHOW STAGES; STAGE_NAME URL STATUS COMMENT -my_ext_stage s3://load/files/ DISABLED -my_ext_stage2 s3://load/files/ DISABLED -my_ext_stage3 s3://load/files/ ENABLED -my_ext_stage1 s3://load/files2/ DISABLED -my_ext_stage4 s3://load/files/ ENABLED user stage +my_ext_stage s3://bucket/files/ DISABLED +my_ext_stage2 s3://bucket/files/ DISABLED +my_ext_stage3 s3://bucket/files/ DISABLED +my_ext_stage1 s3://load/files2/ DISABLED +my_ext_stage4 s3://bucket/files/ DISABLED user stage SHOW STAGES like 'my_ext_stage1'; STAGE_NAME URL STATUS COMMENT -my_ext_stage1 s3://load/files2/ DISABLED +my_ext_stage1 s3://load/files2/ DISABLED DROP STAGE my_ext_stage5; internal error: the stage my_ext_stage5 not exists DROP STAGE if exists my_ext_stage5; @@ -102,26 +113,26 @@ DROP STAGE my_ext_stage3; DROP STAGE my_ext_stage4; drop stage if exists aws_stage; drop stage if exists local_stage; -create stage aws_stage URL= 's3.us-east-2.amazonaws.com' CREDENTIALS={ 'access_key_id' = 'AKIAYOFAMAB', 'secret_access_key' = '7OtGNgIwlkBVwyL9rV', 'bucket' = 'hn-test2', 'region' = 'us-east-2', 'compression' = 'none'}; +create stage aws_stage URL= 's3://hn-test2/a/b/c' CREDENTIALS={ 'AWS_KEY_ID' = 'AKIAYOFAMAB', 'AWS_SECRET_KEY' = '7OtGNgIwlkBVwyL9rV', 'AWS_REGION' = 'us-east-2', 'provider'='minio', 'compression' = 'none'}; show stages; STAGE_NAME URL STATUS COMMENT -aws_stage s3.us-east-2.amazonaws.com DISABLED +aws_stage s3://hn-test2/a/b/c DISABLED alter stage aws_stage set enable=TRUE; show stages; STAGE_NAME URL STATUS COMMENT -aws_stage s3.us-east-2.amazonaws.com ENABLED -alter stage if exists aws_stage set URL= 's3.us-east-1.amazonaws.com'; +aws_stage s3://hn-test2/a/b/c ENABLED +alter stage if exists aws_stage set URL= 's3://bucket2/d/e/f/'; show stages; STAGE_NAME URL STATUS COMMENT -aws_stage s3.us-east-1.amazonaws.com ENABLED -alter stage if exists aws_stage set CREDENTIALS={ 'bucket' = 'hn-test2', 'region' = 'us-east-1'}; -select stage_name,url,stage_credentials,stage_status,comment from mo_catalog.mo_stages; -stage_name url stage_credentials stage_status comment -aws_stage s3.us-east-1.amazonaws.com *7EF9F8E04FF86741707B29BD325543FA2168F6D5 enabled +aws_stage s3://bucket2/d/e/f/ ENABLED +alter stage if exists aws_stage set CREDENTIALS={ 'AWS_REGION' = 'us-east-1'}; +select * from mo_catalog.mo_stages; +stage_id stage_name url stage_credentials stage_status created_time comment +33 aws_stage s3://bucket2/d/e/f/ AWS_REGION=us-east-1 enabled 2024-08-21 10:45:46 alter stage aws_stage set comment='comment1'; show stages; STAGE_NAME URL STATUS COMMENT -aws_stage s3.us-east-1.amazonaws.com ENABLED comment1 +aws_stage s3://bucket2/d/e/f/ ENABLED comment1 drop stage aws_stage; CREATE TABLE stage_table( R_REGIONKEY INTEGER NOT NULL, @@ -135,31 +146,26 @@ insert into stage_table values (2,"ASIA","ges. thinly even pinto beans ca"), (3,"EUROPE","ly final courts cajole furiously final excuse"), (4,"MIDDLE EAST","uickly special accounts cajole carefully blithely close requests. carefully final asymptotes haggle furiousl"); -create stage local_stage URL= '$resources/into_outfile' ENABLE=TRUE comment='local stage configure'; +create stage local_stage URL= 'file:///$resources/into_outfile/stage' comment='local stage configure'; select stage_name,stage_status,comment from mo_catalog.mo_stages; stage_name stage_status comment -local_stage enabled local stage configure -select * from stage_table into outfile 'local_stage:/stage_table.csv'; +local_stage disabled local stage configure +select * from stage_table into outfile 'stage://local_stage/stage_table.csv'; drop stage local_stage; show stages; STAGE_NAME URL STATUS COMMENT -create stage local_stage URL= '$resources/into_outfile' ENABLE=FALSE; -select * from stage_table into outfile 'local_stage:/stage_table.csv'; -internal error: stage 'local_stage' is invalid, please check -select * from stage_table into outfile '$resources/into_outfile/stage_table00.csv'; -alter stage local_stage set ENABLE=TRUE; -select * from stage_table into outfile '$resources/into_outfile/stage_table01.csv'; -internal error: stage exists, please try to check and use a stage instead +create stage local_stage URL= 'file:///$resources/into_outfile'; +create stage sub_local_stage URL= 'stage://local_stage/stage/'; +select * from stage_table into outfile 'stage://sub_local_stage/substage_table.csv'; drop stage local_stage; -select * from stage_table into outfile '$resources/into_outfile/stage_table02.csv'; -create stage local_stage URL= '$resources/into_outfile' ENABLE=FALSE; -alter stage local_stage set ENABLE=TRUE; -select stage_name,stage_status,comment from mo_catalog.mo_stages; -stage_name stage_status comment -local_stage enabled -select * from stage_table into outfile 'local_stage:/stage_table1.csv'; +drop stage sub_local_stage; +select * from stage_table into outfile '$resources/into_outfile/stage/stage_table00.csv'; +select * from stage_table into outfile '$resources/into_outfile/stage/stage_table01.csv'; +select * from stage_table into outfile '$resources/into_outfile/stage/stage_table02.csv'; +create stage local_stage URL= 'file:///$resources/into_outfile/stage'; +select * from stage_table into outfile 'stage://local_stage/stage_table1.csv'; truncate table stage_table; -load data infile '$resources/into_outfile/stage_table1.csv' into table stage_table fields terminated by ',' ignore 1 lines; +load data infile '$resources/into_outfile/stage/stage_table1.csv' into table stage_table fields terminated by ',' ignore 1 lines; select r_name from stage_table; r_name AFRICA @@ -167,13 +173,13 @@ AMERICA ASIA EUROPE MIDDLE EAST -alter stage local_stage set URL= '$resources/into_outfile_2'; +alter stage local_stage set URL= 'file:///$resources/into_outfile_2/stage'; select stage_name,stage_status,comment from mo_catalog.mo_stages; stage_name stage_status comment -local_stage enabled -select * from stage_table into outfile 'local_stage:/stage_table2.csv'; +local_stage disabled +select * from stage_table into outfile 'stage://local_stage/stage_table2.csv'; truncate table stage_table; -load data infile '$resources/into_outfile_2/stage_table2.csv' into table stage_table fields terminated by ',' ignore 1 lines; +load data infile '$resources/into_outfile_2/stage/stage_table2.csv' into table stage_table fields terminated by ',' ignore 1 lines; select r_name from stage_table; r_name AFRICA @@ -184,29 +190,23 @@ MIDDLE EAST alter stage local_stage set comment = 'new comment'; select stage_name,stage_status,comment from mo_catalog.mo_stages; stage_name stage_status comment -local_stage enabled new comment -create stage local_stage URL= '$resources/into_outfile_2' ENABLE=FALSE; -internal error: the stage local_stage exists -alter stage local_stage set URL= '$resources/into_outfile_2' comment='alter comment'; -internal error: at most one option at a time -drop stage local_stage; +local_stage disabled new comment drop stage if exists local_stage; -create stage local_stage URL= '$resources/into_outfile' ENABLE=TRUE comment='local stage configure'; -select * from stage_table into outfile 'local_stage:/stage_table3.csv'; +create stage local_stage URL= 'file:///$resources/into_outfile/stage' comment='local stage configure'; +select * from stage_table into outfile 'stage://local_stage/stage_table3.csv'; drop stage local_stage; -create stage if not exists local_stage URL= '$resources/into_outfile' ENABLE=TRUE comment='local stage configure'; -create stage if not exists local_stage URL= '$resources/into_outfile2' ENABLE=FALSE; +create stage if not exists local_stage URL= 'file:///$resources/into_outfile/stage' comment='local stage configure'; +create stage if not exists local_stage URL= 'file:///$resources/into_outfile2/stage'; select stage_name,stage_status,comment from mo_catalog.mo_stages; stage_name stage_status comment -local_stage enabled local stage configure -alter stage if exists l_stage set ENABLE=TRUE; +local_stage disabled local stage configure create user "stage_user" identified by '123456'; create role s_role; grant all on table *.* to s_role; grant all on account * to s_role; grant all on database *.* to s_role; grant s_role to stage_user; -create stage local_stage URL= '$resources/into_outfile' ENABLE=TRUE comment='local stage configure'; +create stage local_stage URL= 'file:///$resources/into_outfile/stage' comment='local stage configure'; internal error: do not have privilege to execute the statement create database sdb; use sdb; @@ -222,8 +222,6 @@ insert into stage_table values (2,"ASIA","ges. thinly even pinto beans ca"), (3,"EUROPE","ly final courts cajole furiously final excuse"), (4,"MIDDLE EAST","uickly special accounts cajole carefully blithely close requests. carefully final asymptotes haggle furiousl"); -alter stage local_stage set ENABLE=FALSE; -internal error: do not have privilege to execute the statement drop stage stage_user; internal error: do not have privilege to execute the statement drop database sdb; @@ -231,7 +229,8 @@ drop user stage_user; drop role s_role; drop account if exists stage_account; create account `stage_account` ADMIN_NAME 'admin' IDENTIFIED BY '123456'; -create stage local_stage URL= '$resources/into_outfile' ENABLE=TRUE comment='local stage configure'; +create stage local_stage URL= 'file:///$resources/' comment='local stage configure'; +create stage sub_local_stage URL= 'stage://local_stage/into_outfile/stage' comment='sub local stage configure'; create database sdb; use sdb; CREATE TABLE stage_table( @@ -248,9 +247,39 @@ insert into stage_table values (4,"MIDDLE EAST","uickly special accounts cajole carefully blithely close requests. carefully final asymptotes haggle furiousl"); select stage_name,stage_status,comment from mo_catalog.mo_stages; stage_name stage_status comment -local_stage enabled local stage configure -select * from stage_table into outfile 'local_stage:/stage_table5.csv'; +local_stage disabled local stage configure +sub_local_stage disabled sub local stage configure +select * from stage_table into outfile 'stage://sub_local_stage/stage_table5.csv'; +CREATE TABLE stage_infile_table( +R_REGIONKEY INTEGER NOT NULL, +R_NAME CHAR(25) NOT NULL, +R_COMMENT VARCHAR(152), +PRIMARY KEY (R_REGIONKEY) +); +load data infile 'stage://sub_local_stage/stage_table5.csv' into table stage_infile_table fields terminated by ',' IGNORE 1 LINES; +select * from stage_infile_table; +r_regionkey r_name r_comment +0 AFRICA lar deposits. blithely final packages cajole. regular waters are final requests. regular accounts are according to +1 AMERICA hs use ironic, even requests. s +2 ASIA ges. thinly even pinto beans ca +3 EUROPE ly final courts cajole furiously final excuse +4 MIDDLE EAST uickly special accounts cajole carefully blithely close requests. carefully final asymptotes haggle furiousl +CREATE EXTERNAL TABLE stage_ext_table( +R_REGIONKEY INTEGER NOT NULL, +R_NAME CHAR(25) NOT NULL, +R_COMMENT VARCHAR(152) +) INFILE 'stage://sub_local_stage/stage_table*.csv' fields terminated by ',' IGNORE 1 LINES; +select count(*) from stage_ext_table; +count(*) +35 +select count(*) from stage_list('stage://sub_local_stage/') as f; +count(*) +8 +select count(*) from stage_list('stage://sub_local_stage/stage_table*.csv') as f; +count(*) +7 drop stage local_stage; +drop stage sub_local_stage; drop database sdb; create user "stage_user02" identified by '123456'; create role s_role; @@ -258,12 +287,10 @@ grant all on table *.* to s_role; grant all on account * to s_role; grant all on database *.* to s_role; grant s_role to stage_user02; -create stage local_stage URL= '$resources/into_outfile' ENABLE=TRUE; +create stage local_stage URL= 'file:///$resources/into_outfile/stage'; internal error: do not have privilege to execute the statement show stages; STAGE_NAME URL STATUS COMMENT -alter stage local_stage set ENABLE=FALSE; -internal error: do not have privilege to execute the statement drop stage local_stage; internal error: do not have privilege to execute the statement drop account stage_account; diff --git a/test/distributed/cases/stage/stage.sql b/test/distributed/cases/stage/stage.sql index b08079d6fccfb..8c3d687acc332 100644 --- a/test/distributed/cases/stage/stage.sql +++ b/test/distributed/cases/stage/stage.sql @@ -1,19 +1,30 @@ -CREATE STAGE my_ext_stage URL='s3://load/files/'; -CREATE STAGE my_ext_stage URL='s3://load/files/'; -CREATE STAGE if not exists my_ext_stage URL='s3://load/files/'; -SELECT stage_name from mo_catalog.mo_stages; +CREATE STAGE invalid_path_stage URL='/path/to'; +CREATE STAGE invalid_unknown_protocol_stage URL='minio:///path/to'; -CREATE STAGE my_ext_stage1 URL='s3://load/files/' CREDENTIALS={'AWS_KEY_ID'='1a2b3c' ,'AWS_SECRET_KEY'='4x5y6z'}; -CREATE STAGE my_ext_stage2 URL='s3://load/files/' CREDENTIALS={'AWS_KEY_ID'='1a2b3c' ,'AWS_SECRET_KEY'='4x5y6z'}; -SELECT stage_name, stage_status from mo_catalog.mo_stages; +CREATE STAGE my_ext_stage URL='s3://bucket/files/'; +CREATE STAGE my_sub_stage URL='stage://my_ext_stage/a/b/c/'; -CREATE STAGE my_ext_stage3 URL='s3://load/files/' CREDENTIALS={'AWS_KEY_ID'='1a2b3c' ,'AWS_SECRET_KEY'='4x5y6z'} ENABLE = TRUE; -SELECT stage_name, stage_status from mo_catalog.mo_stages; +CREATE STAGE my_ext_stage URL='s3://bucket/files/'; +ALTER STAGE my_ext_stage SET URL='abc'; -ALTER STAGE my_ext_stage4 SET URL='s3://load/files2/'; -ALTER STAGE if exists my_ext_stage4 SET URL='s3://load/files2/'; -ALTER STAGE my_ext_stage1 SET URL='s3://load/files2/' CREDENTIALS={'AWS_KEY_ID'='1a2b3d' ,'AWS_SECRET_KEY'='4x5y6z'}; -ALTER STAGE my_ext_stage1 SET URL='s3://load/files2/'; + +CREATE STAGE if not exists my_ext_stage URL='s3://bucket/files/'; +-- @ignore:0,5 +SELECT * from mo_catalog.mo_stages; + +CREATE STAGE my_ext_stage1 URL='s3://bucket1/files/' CREDENTIALS={'AWS_KEY_ID'='1a2b3c' ,'AWS_SECRET_KEY'='4x5y6z', 'AWS_REGION'='us-east-2', 'PROVIDER'='minio'}; +CREATE STAGE my_ext_stage2 URL='s3://bucket2/files/' CREDENTIALS={'AWS_KEY_ID'='1a2b3c' ,'AWS_SECRET_KEY'='4x5y6z', 'AWS_REGION'='us-east-2', 'PROVIDER'='minio'}; +-- @ignore:0,5 +SELECT * from mo_catalog.mo_stages; + +CREATE STAGE my_ext_stage3 URL='s3://bucket3/files/' CREDENTIALS={'AWS_KEY_ID'='1a2b3c' ,'AWS_SECRET_KEY'='4x5y6z', 'AWS_REGION'='us-east-2', 'PROVIDER'='minio'}; +-- @ignore:0,5 +SELECT * from mo_catalog.mo_stages; + +ALTER STAGE my_ext_stage4 SET URL='s3://bucket4/files2/'; +ALTER STAGE if exists my_ext_stage4 SET URL='s3://bucket4/files2/'; +ALTER STAGE my_ext_stage1 SET URL='s3://bucket2/files2/' CREDENTIALS={'AWS_KEY_ID'='1a2b3d' ,'AWS_SECRET_KEY'='4x5y6z'}; +ALTER STAGE my_ext_stage1 SET URL='s3://bucket2/files2/'; ALTER STAGE my_ext_stage1 SET CREDENTIALS={'AWS_KEY_ID'='1a2b3d' ,'AWS_SECRET_KEY'='4x5y6z'}; DROP STAGE my_ext_stage5; @@ -22,15 +33,18 @@ DROP STAGE my_ext_stage; DROP STAGE my_ext_stage1; DROP STAGE my_ext_stage2; DROP STAGE my_ext_stage3; +DROP STAGE my_sub_stage; -CREATE STAGE my_ext_stage URL='s3://load/files/'; -SELECT stage_name from mo_catalog.mo_stages; +CREATE STAGE my_ext_stage URL='s3://bucket/files/'; +-- @ignore:0,5 +SELECT * from mo_catalog.mo_stages; create account default_1 ADMIN_NAME admin IDENTIFIED BY '111111'; -- @session:id=1&user=default_1:admin&password=111111 -CREATE STAGE my_ext_stage1 URL='s3://load/files/' CREDENTIALS={'AWS_KEY_ID'='1a2b3c' ,'AWS_SECRET_KEY'='4x5y6z'}; -CREATE STAGE my_ext_stage2 URL='s3://load/files/' CREDENTIALS={'AWS_KEY_ID'='1a2b3c' ,'AWS_SECRET_KEY'='4x5y6z'}; -SELECT stage_name, stage_status from mo_catalog.mo_stages; +CREATE STAGE my_ext_stage1 URL='s3://bucket1/files/' CREDENTIALS={'AWS_KEY_ID'='1a2b3c' ,'AWS_SECRET_KEY'='4x5y6z', 'AWS_REGION'='us-east-2', 'PROVIDER'='minio'}; +CREATE STAGE my_ext_stage2 URL='s3://bucket2/files/' CREDENTIALS={'AWS_KEY_ID'='1a2b3c' ,'AWS_SECRET_KEY'='4x5y6z', 'AWS_REGION'='us-east-2', 'PROVIDER'='minio'}; +-- @ignore:0,5 +SELECT * from mo_catalog.mo_stages; DROP STAGE my_ext_stage1; DROP STAGE my_ext_stage2; -- @session @@ -38,17 +52,17 @@ drop account default_1; drop stage my_ext_stage; -CREATE STAGE my_ext_stage URL='s3://load/files/'; -CREATE STAGE my_ext_stage URL='s3://load/files/'; -CREATE STAGE if not exists my_ext_stage URL='s3://load/files/'; +CREATE STAGE my_ext_stage URL='s3://bucket/files/'; +CREATE STAGE my_ext_stage URL='s3://bucket/files/'; +CREATE STAGE if not exists my_ext_stage URL='s3://bucket/files/'; SHOW STAGES; -CREATE STAGE my_ext_stage1 URL='s3://load/files/' CREDENTIALS={'AWS_KEY_ID'='1a2b3c' ,'AWS_SECRET_KEY'='4x5y6z'}; -CREATE STAGE my_ext_stage2 URL='s3://load/files/' CREDENTIALS={'AWS_KEY_ID'='1a2b3c' ,'AWS_SECRET_KEY'='4x5y6z'}; +CREATE STAGE my_ext_stage1 URL='s3://bucket/files/' CREDENTIALS={'AWS_KEY_ID'='1a2b3c' ,'AWS_SECRET_KEY'='4x5y6z'}; +CREATE STAGE my_ext_stage2 URL='s3://bucket/files/' CREDENTIALS={'AWS_KEY_ID'='1a2b3c' ,'AWS_SECRET_KEY'='4x5y6z'}; SHOW STAGES; -CREATE STAGE my_ext_stage3 URL='s3://load/files/' CREDENTIALS={'AWS_KEY_ID'='1a2b3c' ,'AWS_SECRET_KEY'='4x5y6z'} ENABLE = TRUE; -CREATE STAGE my_ext_stage4 URL='s3://load/files/' CREDENTIALS={'AWS_KEY_ID'='1a2b3c' ,'AWS_SECRET_KEY'='4x5y6z'} ENABLE = TRUE COMMENT = 'self stage'; +CREATE STAGE my_ext_stage3 URL='s3://bucket/files/' CREDENTIALS={'AWS_KEY_ID'='1a2b3c' ,'AWS_SECRET_KEY'='4x5y6z'}; +CREATE STAGE my_ext_stage4 URL='s3://bucket/files/' CREDENTIALS={'AWS_KEY_ID'='1a2b3c' ,'AWS_SECRET_KEY'='4x5y6z'} COMMENT = 'self stage'; SHOW STAGES; SHOW STAGES like 'my_ext_stage3'; @@ -72,14 +86,15 @@ DROP STAGE my_ext_stage4; -- create stage aws,cos,aliyun ,now not support select outfile s3 drop stage if exists aws_stage; drop stage if exists local_stage; -create stage aws_stage URL= 's3.us-east-2.amazonaws.com' CREDENTIALS={ 'access_key_id' = 'AKIAYOFAMAB', 'secret_access_key' = '7OtGNgIwlkBVwyL9rV', 'bucket' = 'hn-test2', 'region' = 'us-east-2', 'compression' = 'none'}; +create stage aws_stage URL= 's3://hn-test2/a/b/c' CREDENTIALS={ 'AWS_KEY_ID' = 'AKIAYOFAMAB', 'AWS_SECRET_KEY' = '7OtGNgIwlkBVwyL9rV', 'AWS_REGION' = 'us-east-2', 'provider'='minio', 'compression' = 'none'}; show stages; alter stage aws_stage set enable=TRUE; show stages; -alter stage if exists aws_stage set URL= 's3.us-east-1.amazonaws.com'; +alter stage if exists aws_stage set URL= 's3://bucket2/d/e/f/'; show stages; -alter stage if exists aws_stage set CREDENTIALS={ 'bucket' = 'hn-test2', 'region' = 'us-east-1'}; -select stage_name,url,stage_credentials,stage_status,comment from mo_catalog.mo_stages; +alter stage if exists aws_stage set CREDENTIALS={ 'AWS_REGION' = 'us-east-1'}; +-- @ignore:0,5 +select * from mo_catalog.mo_stages; alter stage aws_stage set comment='comment1'; show stages; drop stage aws_stage; @@ -98,63 +113,55 @@ insert into stage_table values (3,"EUROPE","ly final courts cajole furiously final excuse"), (4,"MIDDLE EAST","uickly special accounts cajole carefully blithely close requests. carefully final asymptotes haggle furiousl"); -- create stage local disk -create stage local_stage URL= '$resources/into_outfile' ENABLE=TRUE comment='local stage configure'; +create stage local_stage URL= 'file:///$resources/into_outfile/stage' comment='local stage configure'; select stage_name,stage_status,comment from mo_catalog.mo_stages; -select * from stage_table into outfile 'local_stage:/stage_table.csv'; +select * from stage_table into outfile 'stage://local_stage/stage_table.csv'; drop stage local_stage; show stages; -- url params error -- create stage local_stage URL= '$resources/abc' ENABLE=TRUE; --- select * from stage_table into outfile 'local_stage:/stage_table.csv'; +-- select * from stage_table into outfile 'stage://local_stage/stage_table.csv'; -- drop stage local_stage; --- enable is false -create stage local_stage URL= '$resources/into_outfile' ENABLE=FALSE; -select * from stage_table into outfile 'local_stage:/stage_table.csv'; +-- output to sub-stage file +create stage local_stage URL= 'file:///$resources/into_outfile'; +create stage sub_local_stage URL= 'stage://local_stage/stage/'; +select * from stage_table into outfile 'stage://sub_local_stage/substage_table.csv'; +drop stage local_stage; +drop stage sub_local_stage; -- select outfile without stage -select * from stage_table into outfile '$resources/into_outfile/stage_table00.csv'; -alter stage local_stage set ENABLE=TRUE; -select * from stage_table into outfile '$resources/into_outfile/stage_table01.csv'; -drop stage local_stage; -select * from stage_table into outfile '$resources/into_outfile/stage_table02.csv'; +select * from stage_table into outfile '$resources/into_outfile/stage/stage_table00.csv'; +select * from stage_table into outfile '$resources/into_outfile/stage/stage_table01.csv'; +select * from stage_table into outfile '$resources/into_outfile/stage/stage_table02.csv'; -- alter stage params: enable/URL/comment -create stage local_stage URL= '$resources/into_outfile' ENABLE=FALSE; -alter stage local_stage set ENABLE=TRUE; -select stage_name,stage_status,comment from mo_catalog.mo_stages; -select * from stage_table into outfile 'local_stage:/stage_table1.csv'; +create stage local_stage URL= 'file:///$resources/into_outfile/stage'; +select * from stage_table into outfile 'stage://local_stage/stage_table1.csv'; truncate table stage_table; -load data infile '$resources/into_outfile/stage_table1.csv' into table stage_table fields terminated by ',' ignore 1 lines; +load data infile '$resources/into_outfile/stage/stage_table1.csv' into table stage_table fields terminated by ',' ignore 1 lines; select r_name from stage_table; -alter stage local_stage set URL= '$resources/into_outfile_2'; +alter stage local_stage set URL= 'file:///$resources/into_outfile_2/stage'; select stage_name,stage_status,comment from mo_catalog.mo_stages; -select * from stage_table into outfile 'local_stage:/stage_table2.csv'; +select * from stage_table into outfile 'stage://local_stage/stage_table2.csv'; truncate table stage_table; -load data infile '$resources/into_outfile_2/stage_table2.csv' into table stage_table fields terminated by ',' ignore 1 lines; +load data infile '$resources/into_outfile_2/stage/stage_table2.csv' into table stage_table fields terminated by ',' ignore 1 lines; select r_name from stage_table; alter stage local_stage set comment = 'new comment'; select stage_name,stage_status,comment from mo_catalog.mo_stages; --- abnormal test -create stage local_stage URL= '$resources/into_outfile_2' ENABLE=FALSE; -alter stage local_stage set URL= '$resources/into_outfile_2' comment='alter comment'; -drop stage local_stage; ---create stage local_stage URL= '$resources/into_outfile_2' ENABLE=ffalse; - -- select outfile file exists drop stage if exists local_stage; -create stage local_stage URL= '$resources/into_outfile' ENABLE=TRUE comment='local stage configure'; -select * from stage_table into outfile 'local_stage:/stage_table3.csv'; +create stage local_stage URL= 'file:///$resources/into_outfile/stage' comment='local stage configure'; +select * from stage_table into outfile 'stage://local_stage/stage_table3.csv'; --select * from stage_table into outfile 'local_stage:/stage_table3.csv'; drop stage local_stage; -- if exists -create stage if not exists local_stage URL= '$resources/into_outfile' ENABLE=TRUE comment='local stage configure'; -create stage if not exists local_stage URL= '$resources/into_outfile2' ENABLE=FALSE; +create stage if not exists local_stage URL= 'file:///$resources/into_outfile/stage' comment='local stage configure'; +create stage if not exists local_stage URL= 'file:///$resources/into_outfile2/stage'; select stage_name,stage_status,comment from mo_catalog.mo_stages; -alter stage if exists l_stage set ENABLE=TRUE; -- privs confirm create user "stage_user" identified by '123456'; @@ -164,7 +171,7 @@ grant all on account * to s_role; grant all on database *.* to s_role; grant s_role to stage_user; -- @session:id=2&user=sys:stage_user:s_role&password=123456 -create stage local_stage URL= '$resources/into_outfile' ENABLE=TRUE comment='local stage configure'; +create stage local_stage URL= 'file:///$resources/into_outfile/stage' comment='local stage configure'; create database sdb; use sdb; CREATE TABLE stage_table( @@ -179,7 +186,6 @@ insert into stage_table values (2,"ASIA","ges. thinly even pinto beans ca"), (3,"EUROPE","ly final courts cajole furiously final excuse"), (4,"MIDDLE EAST","uickly special accounts cajole carefully blithely close requests. carefully final asymptotes haggle furiousl"); -alter stage local_stage set ENABLE=FALSE; drop stage stage_user; drop database sdb; -- @session @@ -189,7 +195,8 @@ drop role s_role; drop account if exists stage_account; create account `stage_account` ADMIN_NAME 'admin' IDENTIFIED BY '123456'; -- @session:id=3&user=stage_account:admin&password=123456 -create stage local_stage URL= '$resources/into_outfile' ENABLE=TRUE comment='local stage configure'; +create stage local_stage URL= 'file:///$resources/' comment='local stage configure'; +create stage sub_local_stage URL= 'stage://local_stage/into_outfile/stage' comment='sub local stage configure'; create database sdb; use sdb; CREATE TABLE stage_table( @@ -205,8 +212,36 @@ insert into stage_table values (3,"EUROPE","ly final courts cajole furiously final excuse"), (4,"MIDDLE EAST","uickly special accounts cajole carefully blithely close requests. carefully final asymptotes haggle furiousl"); select stage_name,stage_status,comment from mo_catalog.mo_stages; -select * from stage_table into outfile 'local_stage:/stage_table5.csv'; + +-- write to sub stage +select * from stage_table into outfile 'stage://sub_local_stage/stage_table5.csv'; + +-- read from stage file +CREATE TABLE stage_infile_table( +R_REGIONKEY INTEGER NOT NULL, +R_NAME CHAR(25) NOT NULL, +R_COMMENT VARCHAR(152), +PRIMARY KEY (R_REGIONKEY) +); +load data infile 'stage://sub_local_stage/stage_table5.csv' into table stage_infile_table fields terminated by ',' IGNORE 1 LINES; +select * from stage_infile_table; + +-- read from external table and sub stage +CREATE EXTERNAL TABLE stage_ext_table( +R_REGIONKEY INTEGER NOT NULL, +R_NAME CHAR(25) NOT NULL, +R_COMMENT VARCHAR(152) +) INFILE 'stage://sub_local_stage/stage_table*.csv' fields terminated by ',' IGNORE 1 LINES; +select count(*) from stage_ext_table; + +-- list the stage directory +select count(*) from stage_list('stage://sub_local_stage/') as f; + +-- list the stage directory with wildcard +select count(*) from stage_list('stage://sub_local_stage/stage_table*.csv') as f; + drop stage local_stage; +drop stage sub_local_stage; drop database sdb; create user "stage_user02" identified by '123456'; create role s_role; @@ -217,9 +252,8 @@ grant s_role to stage_user02; -- @session -- @session:id=4&user=stage_account:stage_user02:s_role&password=123456 -create stage local_stage URL= '$resources/into_outfile' ENABLE=TRUE; +create stage local_stage URL= 'file:///$resources/into_outfile/stage'; show stages; -alter stage local_stage set ENABLE=FALSE; drop stage local_stage; -- @session drop account stage_account;