chore: optimize runs api

pkg/domain/constant/global.go

@@ -19,4 +19,9 @@ const (
 	RepoCacheTTL            = 60 * time.Minute
 	RunTimeOut              = 60 * time.Minute
 	DefaultWorkloadSig      = "kusion.io/is-workload"
+	ResourcePageDefault     = 1
+	ResourcePageSizeDefault = 100
+	ResourcePageSizeLarge   = 1000
+	RunPageDefault          = 1
+	RunPageSizeDefault      = 10
 )

pkg/domain/constant/organization.go

@@ -22,6 +22,7 @@ var (
 	ErrBackendNil              = errors.New("backend is nil")
 	ErrBackendNameEmpty        = errors.New("backend must have a name")
 	ErrBackendTypeEmpty        = errors.New("backend must have a type")
+	ErrDefaultBackendNotSet    = errors.New("default backend not set properly")
 	ErrAppConfigHasNilStack    = errors.New("appConfig has nil stack")
 	ErrInvalidOrganizationName = errors.New("organization name can only have alphanumeric characters and underscores with [a-zA-Z0-9_]")
 	ErrInvalidOrganizationID   = errors.New("the organization ID should be a uuid")

pkg/domain/entity/run.go

@@ -22,7 +22,8 @@ type Run struct {
 	Status constant.RunStatus `yaml:"status" json:"status"`
 	// Result is the result of the run.
 	Result string `yaml:"result" json:"result"`
-	// Result RunResult `yaml:"result" json:"result"`
+	// Trace is the trace of the run.
+	Trace string `yaml:"trace" json:"trace"`
 	// Logs is the logs of the run.
 	Logs string `yaml:"logs" json:"logs"`
 	// CreationTimestamp is the timestamp of the created for the run.
@@ -44,9 +45,15 @@ type RunResult struct {
 }
 
 type RunFilter struct {
-	ProjectID uint
-	StackID   uint
-	Workspace string
+	ProjectID  uint
+	StackID    uint
+	Workspace  string
+	Pagination *Pagination
+}
+
+type RunListResult struct {
+	Runs  []*Run
+	Total int
 }
 
 // Validate checks if the run is valid.

pkg/domain/entity/types.go

@@ -0,0 +1,6 @@
+package entity
+
+type Pagination struct {
+	Page     int `json:"page"`
+	PageSize int `json:"pageSize"`
+}

pkg/domain/repository/repository.go

@@ -150,5 +150,5 @@ type RunRepository interface {
 	// Get retrieves a run by its ID.
 	Get(ctx context.Context, id uint) (*entity.Run, error)
 	// List retrieves all existing run.
-	List(ctx context.Context, filter *entity.RunFilter) ([]*entity.Run, error)
+	List(ctx context.Context, filter *entity.RunFilter) (*entity.RunListResult, error)
 }

pkg/domain/response/resource.go

@@ -0,0 +1,12 @@
+package response
+
+import (
+	"kusionstack.io/kusion/pkg/domain/entity"
+)
+
+type PaginatedResourceResponse struct {
+	Resources   []*entity.Resource `json:"resources"`
+	Total       int                `json:"total"`
+	CurrentPage int                `json:"currentPage"`
+	PageSize    int                `json:"pageSize"`
+}

pkg/domain/response/run.go

@@ -0,0 +1,12 @@
+package response
+
+import (
+	"kusionstack.io/kusion/pkg/domain/entity"
+)
+
+type PaginatedRunResponse struct {
+	Runs        []*entity.Run `json:"runs"`
+	Total       int           `json:"total"`
+	CurrentPage int           `json:"currentPage"`
+	PageSize    int           `json:"pageSize"`
+}

pkg/infra/persistence/run.go

@@ -96,17 +96,24 @@ func (r *runRepository) Get(ctx context.Context, id uint) (*entity.Run, error) {
 }
 
 // List retrieves all runs.
-func (r *runRepository) List(ctx context.Context, filter *entity.RunFilter) ([]*entity.Run, error) {
+func (r *runRepository) List(ctx context.Context, filter *entity.RunFilter) (*entity.RunListResult, error) {
 	var dataModel []RunModel
 	runEntityList := make([]*entity.Run, 0)
 	pattern, args := GetRunQuery(filter)
-	result := r.db.WithContext(ctx).
+	searchResult := r.db.WithContext(ctx).
 		Preload("Stack").Preload("Stack.Project").
 		Joins("JOIN stack ON stack.id = run.stack_id").
 		Joins("JOIN project ON project.id = stack.project_id").
 		Joins("JOIN workspace ON workspace.name = run.workspace").
-		Where(pattern, args...).
-		Find(&dataModel)
+		Where(pattern, args...)
+
+		// Get total rows
+	var totalRows int64
+	searchResult.Model(dataModel).Count(&totalRows)
+
+	// Fetch paginated data from searchResult with offset and limit
+	offset := (filter.Pagination.Page - 1) * filter.Pagination.PageSize
+	result := searchResult.Offset(offset).Limit(filter.Pagination.PageSize).Find(&dataModel)
 	if result.Error != nil {
 		return nil, result.Error
 	}
@@ -117,5 +124,8 @@ func (r *runRepository) List(ctx context.Context, filter *entity.RunFilter) ([]*
 		}
 		runEntityList = append(runEntityList, runEntity)
 	}
-	return runEntityList, nil
+	return &entity.RunListResult{
+		Runs:  runEntityList,
+		Total: int(totalRows),
+	}, nil
 }

pkg/infra/persistence/run_model.go

@@ -24,6 +24,8 @@ type RunModel struct {
 	Result string
 	// Logs is the logs of the run.
 	Logs string
+	// Trace is the trace of the run.
+	Trace string
 }
 
 // The TableName method returns the name of the database table that the struct is mapped to.
@@ -53,13 +55,13 @@ func (m *RunModel) ToEntity() (*entity.Run, error) {
 	}
 
 	return &entity.Run{
-		ID:        m.ID,
-		Type:      runType,
-		Stack:     stackEntity,
-		Workspace: m.Workspace,
-		Status:    runStatus,
-		Result:    m.Result,
-		// Result:            entity.RunResult{},
+		ID:                m.ID,
+		Type:              runType,
+		Stack:             stackEntity,
+		Workspace:         m.Workspace,
+		Status:            runStatus,
+		Result:            m.Result,
+		Trace:             m.Trace,
 		Logs:              m.Logs,
 		CreationTimestamp: m.CreatedAt,
 		UpdateTimestamp:   m.UpdatedAt,
@@ -83,6 +85,7 @@ func (m *RunModel) FromEntity(e *entity.Run) error {
 	m.Status = string(e.Status)
 	m.Result = e.Result
 	m.Logs = e.Logs
+	m.Trace = e.Trace
 	m.CreatedAt = e.CreationTimestamp
 	m.UpdatedAt = e.UpdateTimestamp
 

pkg/server/handler/stack/execute_async.go

@@ -76,7 +76,8 @@ func (h *Handler) PreviewStackAsync() http.HandlerFunc {
 			logger.Info("Async preview in progress")
 			var previewChanges any
 			newCtx, cancel := CopyToNewContextWithTimeout(ctx, constant.RunTimeOut)
-			defer cancel() // make sure the context is canceled to free resources
+			defer cancel()            // make sure the context is canceled to free resources
+			defer handleCrash(newCtx) // recover from possible panic
 
 			// update status of the run when exiting the async run
 			defer func() {
@@ -103,14 +104,12 @@ func (h *Handler) PreviewStackAsync() http.HandlerFunc {
 			// Call preview stack
 			changes, err := h.stackManager.PreviewStack(newCtx, params, requestPayload.ImportedResources)
 			if err != nil {
-				// render.Render(w, r, handler.FailureResponse(ctx, err))
 				logger.Error("Error previewing stack", "error", err)
 				return
 			}
 
 			previewChanges, err = stackmanager.ProcessChanges(newCtx, w, changes, params.Format, params.ExecuteParams.Detail)
 			if err != nil {
-				// render.Render(w, r, handler.FailureResponse(ctx, err))
 				logger.Error("Error processing preview changes", "error", err)
 				return
 			}
@@ -180,7 +179,8 @@ func (h *Handler) ApplyStackAsync() http.HandlerFunc {
 			// defer safe.HandleCrash(aciLoggingRecoverHandler(h.aciClient, &req, log))
 			logger.Info("Async apply in progress")
 			newCtx, cancel := CopyToNewContextWithTimeout(ctx, constant.RunTimeOut)
-			defer cancel() // make sure the context is canceled to free resources
+			defer cancel()            // make sure the context is canceled to free resources
+			defer handleCrash(newCtx) // recover from possible panic
 
 			// update status of the run when exiting the async run
 			defer func() {
@@ -206,7 +206,6 @@ func (h *Handler) ApplyStackAsync() http.HandlerFunc {
 					render.Render(w, r, handler.SuccessResponse(ctx, "Dry-run mode enabled, the above resources will be applied if dryrun is set to false"))
 					return
 				} else {
-					// render.Render(w, r, handler.FailureResponse(ctx, err))
 					logger.Error("Error applying stack", "error", err)
 					return
 				}
@@ -282,9 +281,10 @@ func (h *Handler) GenerateStackAsync() http.HandlerFunc {
 			// defer safe.HandleCrash(aciLoggingRecoverHandler(h.aciClient, &req, log))
 			logger.Info("Async generate in progress")
 			newCtx, cancel := CopyToNewContextWithTimeout(ctx, constant.RunTimeOut)
-			var sp *apiv1.Spec
-			defer cancel() // make sure the context is canceled to free resources
+			defer cancel()            // make sure the context is canceled to free resources
+			defer handleCrash(newCtx) // recover from possible panic
 
+			var sp *apiv1.Spec
 			// update status of the run when exiting the async run
 			defer func() {
 				select {
@@ -308,9 +308,8 @@ func (h *Handler) GenerateStackAsync() http.HandlerFunc {
 			}()
 
 			// Call generate stack
-			_, sp, err := h.stackManager.GenerateSpec(newCtx, params)
+			_, sp, err = h.stackManager.GenerateSpec(newCtx, params)
 			if err != nil {
-				// render.Render(w, r, handler.FailureResponse(ctx, err))
 				logger.Error("Error generating stack", "error", err)
 				return
 			}
@@ -375,10 +374,10 @@ func (h *Handler) DestroyStackAsync() http.HandlerFunc {
 
 		// Starts a safe goroutine using given recover handler
 		inBufferZone := h.workerPool.Do(func() {
-			// defer safe.HandleCrash(aciLoggingRecoverHandler(h.aciClient, &req, log))
 			logger.Info("Async destroy in progress")
 			newCtx, cancel := CopyToNewContextWithTimeout(ctx, constant.RunTimeOut)
-			defer cancel() // make sure the context is canceled to free resources
+			defer cancel()            // make sure the context is canceled to free resources
+			defer handleCrash(newCtx) // recover from possible panic
 
 			// update status of the run when exiting the async run
 			defer func() {
@@ -400,11 +399,9 @@ func (h *Handler) DestroyStackAsync() http.HandlerFunc {
 			err = h.stackManager.DestroyStack(newCtx, params, w)
 			if err != nil {
 				if err == stackmanager.ErrDryrunDestroy {
-					// render.Render(w, r, handler.SuccessResponse(ctx, "Dry-run mode enabled, the above resources will be destroyed if dryrun is set to false"))
 					logger.Info("Dry-run mode enabled, the above resources will be destroyed if dryrun is set to false")
 					return
 				} else {
-					// render.Render(w, r, handler.FailureResponse(ctx, err))
 					logger.Error("Error destroying stack", "error", err)
 					return
 				}

pkg/server/handler/stack/run.go

@@ -5,6 +5,7 @@ import (
 	"net/http"
 
 	"github.com/go-chi/render"
+	response "kusionstack.io/kusion/pkg/domain/response"
 	"kusionstack.io/kusion/pkg/server/handler"
 	logutil "kusionstack.io/kusion/pkg/server/util/logging"
 )
@@ -95,18 +96,26 @@ func (h *Handler) ListRuns() http.HandlerFunc {
 		logger := logutil.GetLogger(ctx)
 		logger.Info("Listing runs...")
 
-		projectIDParam := r.URL.Query().Get("projectID")
-		stackIDParam := r.URL.Query().Get("stackID")
-		workspaceParam := r.URL.Query().Get("workspace")
-
-		filter, err := h.stackManager.BuildRunFilter(ctx, projectIDParam, stackIDParam, workspaceParam)
+		query := r.URL.Query()
+		filter, err := h.stackManager.BuildRunFilter(ctx, &query)
 		if err != nil {
 			render.Render(w, r, handler.FailureResponse(ctx, err))
 			return
 		}
 
+		// List runs
 		runEntities, err := h.stackManager.ListRuns(ctx, filter)
-		handler.HandleResult(w, r, ctx, err, runEntities)
+		if err != nil {
+			render.Render(w, r, handler.FailureResponse(ctx, err))
+			return
+		}
+		paginatedResponse := response.PaginatedRunResponse{
+			Runs:        runEntities.Runs,
+			Total:       runEntities.Total,
+			CurrentPage: filter.Pagination.Page,
+			PageSize:    filter.Pagination.PageSize,
+		}
+		handler.HandleResult(w, r, ctx, err, paginatedResponse)
 	}
 }
 

pkg/server/handler/stack/utils.go

@@ -5,6 +5,7 @@ import (
 	"context"
 	"encoding/json"
 	"net/http"
+	"runtime"
 	"strconv"
 	"time"
 
@@ -169,3 +170,20 @@ func CopyToNewContextWithTimeout(ctx context.Context, timeout time.Duration) (co
 	newCtxWithTimeout, cancel := context.WithTimeout(newCtx, timeout)
 	return newCtxWithTimeout, cancel
 }
+
+func logStackTrace(runLogger *httplog.Logger) {
+	buf := make([]byte, 1<<16) // 64KB
+	stackSize := runtime.Stack(buf, true)
+	runLogger.Error("Stack trace:")
+	runLogger.Error(string(buf[:stackSize]))
+}
+
+func handleCrash(ctx context.Context) {
+	if r := recover(); r != nil {
+		logger := logutil.GetLogger(ctx)
+		runLogger := logutil.GetRunLogger(ctx)
+		logger.Error("Recovered from panic during async execution:", "error", r)
+		runLogger.Error("Panic recovered", "error", r)
+		logStackTrace(runLogger)
+	}
+}