working admin endpoints for object operations

api/admin/storage.go

@@ -62,7 +62,7 @@ func CreateBucket(c *gin.Context) {
 
 		return
 	}
-
+	l.Debugf("bucket name: %s", input.BucketName)
 	err = storage.FromGinContext(c).CreateBucket(ctx, input)
 	if err != nil {
 		retErr := fmt.Errorf("unable to create bucket: %w", err)
@@ -194,3 +194,217 @@ func SetBucketLifecycle(c *gin.Context) {
 
 	c.Status(http.StatusOK)
 }
+
+// swagger:operation POST /api/v1/admin/storage/bucket/upload admin UploadObject
+//
+// # Upload an object to a bucket
+//
+// ---
+// produces:
+// - application/json
+// parameters:
+//   - in: body
+//     name: body
+//     description: The object to be uploaded
+//     required: true
+//     schema:
+//     type: object
+//     properties:
+//     bucketName:
+//     type: string
+//     objectName:
+//     type: string
+//     objectData:
+//     type: string
+//
+// security:
+//   - ApiKeyAuth: []
+//
+// responses:
+//
+//	'201':
+//	  description: Successfully uploaded the object
+//	'400':
+//	  description: Invalid request payload
+//	  schema:
+//	    "$ref": "#/definitions/Error"
+//	'500':
+//	  description: Unexpected server error
+//	  schema:
+//	    "$ref": "#/definitions/Error"
+//
+// UploadObject represents the API handler to upload an object to a bucket.
+func UploadObject(c *gin.Context) {
+	l := c.MustGet("logger").(*logrus.Entry)
+	ctx := c.Request.Context()
+
+	l.Debug("platform admin: uploading object")
+
+	// capture body from API request
+	input := new(types.Object)
+
+	err := c.Bind(input)
+	if err != nil {
+		retErr := fmt.Errorf("unable to decode JSON for object %s: %w", input.ObjectName, err)
+
+		util.HandleError(c, http.StatusBadRequest, retErr)
+
+		return
+	}
+	if input.BucketName == "" || input.ObjectName == "" {
+		retErr := fmt.Errorf("bucketName and objectName are required")
+		util.HandleError(c, http.StatusBadRequest, retErr)
+		return
+	}
+	if input.FilePath == "" {
+		retErr := fmt.Errorf("file path is required")
+		util.HandleError(c, http.StatusBadRequest, retErr)
+		return
+	}
+	err = storage.FromGinContext(c).Upload(ctx, input)
+	if err != nil {
+		retErr := fmt.Errorf("unable to upload object: %w", err)
+		util.HandleError(c, http.StatusInternalServerError, retErr)
+		return
+	}
+
+	c.Status(http.StatusCreated)
+}
+
+// swagger:operation GET /api/v1/admin/storage/bucket/download admin DownloadObject
+//
+// # Download an object from a bucket
+//
+// ---
+// produces:
+// - application/json
+// parameters:
+//   - in: query
+//     name: bucketName
+//     description: The name of the bucket
+//     required: true
+//     type: string
+//   - in: query
+//     name: objectName
+//     description: The name of the object
+//     required: true
+//     type: string
+//
+// security:
+//   - ApiKeyAuth: []
+//
+// responses:
+//
+//	'200':
+//	  description: Successfully downloaded the object
+//	'400':
+//	  description: Invalid request payload
+//	  schema:
+//	    "$ref": "#/definitions/Error"
+//	'500':
+//	  description: Unexpected server error
+//	  schema:
+//	    "$ref": "#/definitions/Error"
+//
+// DownloadObject represents the API handler to download an object from a bucket.
+func DownloadObject(c *gin.Context) {
+	l := c.MustGet("logger").(*logrus.Entry)
+	ctx := c.Request.Context()
+
+	l.Debug("platform admin: downloading object")
+
+	// capture body from API request
+	input := new(types.Object)
+
+	err := c.Bind(input)
+	if err != nil {
+		retErr := fmt.Errorf("unable to decode JSON for object %s: %w", input.ObjectName, err)
+
+		util.HandleError(c, http.StatusBadRequest, retErr)
+
+		return
+	}
+	if input.BucketName == "" || input.ObjectName == "" {
+		retErr := fmt.Errorf("bucketName and objectName are required")
+		util.HandleError(c, http.StatusBadRequest, retErr)
+		return
+	}
+	if input.FilePath == "" {
+		retErr := fmt.Errorf("file path is required")
+		util.HandleError(c, http.StatusBadRequest, retErr)
+		return
+	}
+	err = storage.FromGinContext(c).Download(ctx, input)
+	if err != nil {
+		retErr := fmt.Errorf("unable to download object: %w", err)
+		util.HandleError(c, http.StatusInternalServerError, retErr)
+		return
+	}
+
+	c.JSON(http.StatusOK, gin.H{"message": fmt.Sprintf("File has been downloaded to %s", input.FilePath)})
+}
+
+// swagger:operation GET /api/v1/admin/storage/presign admin GetPresignedURL
+//
+// # Generate a presigned URL for an object
+//
+// ---
+// produces:
+// - application/json
+// parameters:
+//   - in: query
+//     name: bucketName
+//     description: The name of the bucket
+//     required: true
+//     type: string
+//   - in: query
+//     name: objectName
+//     description: The name of the object
+//     required: true
+//     type: string
+//
+// security:
+//   - ApiKeyAuth: []
+//
+// responses:
+//
+//	'200':
+//	  description: Successfully generated the presigned URL
+//	'400':
+//	  description: Invalid request payload
+//	  schema:
+//	    "$ref": "#/definitions/Error"
+//	'500':
+//	  description: Unexpected server error
+//	  schema:
+//	    "$ref": "#/definitions/Error"
+func GetPresignedURL(c *gin.Context) {
+	l := c.MustGet("logger").(*logrus.Entry)
+	ctx := c.Request.Context()
+
+	l.Debug("platform admin: generating presigned URL")
+
+	// capture query parameters from API request
+	bucketName := c.Query("bucketName")
+	objectName := c.Query("objectName")
+
+	if bucketName == "" || objectName == "" {
+		retErr := fmt.Errorf("bucketName and objectName are required")
+		util.HandleError(c, http.StatusBadRequest, retErr)
+		return
+	}
+
+	input := &types.Object{
+		BucketName: bucketName,
+		ObjectName: objectName,
+	}
+
+	url, err := storage.FromGinContext(c).PresignedGetObject(ctx, input)
+	if err != nil {
+		retErr := fmt.Errorf("unable to generate presigned URL: %w", err)
+		util.HandleError(c, http.StatusInternalServerError, retErr)
+		return
+	}
+
+	c.JSON(http.StatusOK, url)
+}

api/types/storage.go

@@ -15,3 +15,9 @@ type BucketOptions struct {
 	Region        string `json:"region,omitempty"`
 	ObjectLocking bool   `json:"object_locking,omitempty"`
 }
+
+type Object struct {
+	ObjectName string `json:"object_name,omitempty"`
+	BucketName string `json:"bucket_name,omitempty"`
+	FilePath   string `json:"file_path,omitempty"`
+}

docker-compose.yml

@@ -49,9 +49,12 @@ services:
       VELA_OTEL_TRACING_SAMPLER_RATELIMIT_PER_SECOND: 100
       VELA_STORAGE_ENABLE: 'true'
       VELA_STORAGE_DRIVER: minio
-      VELA_STORAGE_ENDPOINT: objectstorage:9000
-      VELA_STORAGE_ACCESS_KEY: minio_access_user
-      VELA_STORAGE_SECRET_KEY: minio_secret_key
+#      VELA_STORAGE_ENDPOINT: objectstorage:9000
+#      VELA_STORAGE_ACCESS_KEY: minio_access_user
+#      VELA_STORAGE_SECRET_KEY: minio_secret_key
+      VELA_STORAGE_ENDPOINT: stage.ttc.toss.target.com
+      VELA_STORAGE_ACCESS_KEY:
+      VELA_STORAGE_SECRET_KEY:
       VELA_STORAGE_USE_SSL: 'false'
       VELA_STORAGE_BUCKET: vela
     env_file:

router/admin.go

@@ -60,12 +60,23 @@ func AdminHandlers(base *gin.RouterGroup) {
 		// Admin step endpoint
 		_admin.PUT("/step", admin.UpdateStep)
 
-		// Admin storage endpoints
-		//_admin.GET("/storage/bucket", admin.)
+		// Admin storage bucket endpoints
+		//_admin.GET("/storage/bucket", admin.ListBuckets)
 		_admin.POST("/storage/bucket", admin.CreateBucket)
 		_admin.PUT("/storage/bucket", admin.SetBucketLifecycle)
 		_admin.DELETE("/storage/bucket", admin.DeleteBucket)
 
+		// Admin storage bucket lifecycle endpoint
+		//_admin.GET("/storage/bucket/lifecycle", admin.)
+		//_admin.POST("/storage/bucket/lifecycle", admin.)
+
+		// Admin storage object endpoints
+		_admin.POST("/storage/object/download", admin.DownloadObject)
+		_admin.POST("/storage/object", admin.UploadObject)
+
+		// Admin storage presign endpoints
+		_admin.GET("/storage/presign", admin.GetPresignedURL)
+
 		// Admin user endpoint
 		_admin.PUT("/user", admin.UpdateUser)
 

storage/minio/delete.go

@@ -2,14 +2,15 @@ package minio
 
 import (
 	"context"
+	api "github.com/go-vela/server/api/types"
 	"github.com/minio/minio-go/v7"
 )
 
 // Delete deletes an object in a bucket in MinIO.
-func (c *MinioClient) Delete(ctx context.Context, bucketName string, objectName string) error {
-	c.Logger.Tracef("deleting objectName: %s from bucketName: %s", objectName, bucketName)
+func (c *MinioClient) Delete(ctx context.Context, object *api.Object) error {
+	c.Logger.Tracef("deleting objectName: %s from bucketName: %s", object.ObjectName, object.BucketName)
 
-	err := c.client.RemoveObject(ctx, bucketName, objectName, minio.RemoveObjectOptions{})
+	err := c.client.RemoveObject(ctx, object.BucketName, object.ObjectName, minio.RemoveObjectOptions{})
 	if err != nil {
 		return err
 	}

storage/minio/download.go

@@ -2,15 +2,29 @@ package minio
 
 import (
 	"context"
+	api "github.com/go-vela/server/api/types"
 	"github.com/minio/minio-go/v7"
-	"io"
 )
 
-func (c *MinioClient) Download(ctx context.Context, bucketName, key string) ([]byte, error) {
-	object, err := c.client.GetObject(ctx, bucketName, key, minio.GetObjectOptions{})
+func (c *MinioClient) Download(ctx context.Context, object *api.Object) error {
+
+	// Check if the directory exists
+	//_, err := os.Stat(object.FilePath)
+	//if os.IsNotExist(err) {
+	//	// Create the directory if it does not exist
+	//	err = os.MkdirAll(object.FilePath, 0755)
+	//	if err != nil {
+	//		return fmt.Errorf("failed to create directory: %w", err)
+	//	}
+	//} else if err != nil {
+	//	return fmt.Errorf("failed to check directory: %w", err)
+	//}
+	err := c.client.FGetObject(ctx, object.BucketName, object.ObjectName, object.FilePath, minio.GetObjectOptions{})
 	if err != nil {
-		return nil, err
+		c.Logger.Errorf("unable to retrive object %s", object.ObjectName)
+		return err
 	}
-	defer object.Close()
-	return io.ReadAll(object)
+
+	c.Logger.Tracef("successfully downloaded object %s to %s", object.ObjectName, object.FilePath)
+	return nil
 }

storage/minio/presigned_get_object.go

@@ -0,0 +1,19 @@
+package minio
+
+import (
+	"context"
+	api "github.com/go-vela/server/api/types"
+	"time"
+)
+
+// PresignedGetObject generates a presigned URL for downloading an object.
+func (c *MinioClient) PresignedGetObject(ctx context.Context, object *api.Object) (string, error) {
+	// Generate presigned URL for downloading the object.
+	// The URL is valid for 7 days.
+	presignedURL, err := c.client.PresignedGetObject(ctx, object.BucketName, object.ObjectName, 7*24*time.Hour, nil)
+	if err != nil {
+		return "", err
+	}
+
+	return presignedURL.String(), nil
+}

storage/minio/upload.go

@@ -1,16 +1,15 @@
 package minio
 
 import (
-	"bytes"
 	"context"
+	api "github.com/go-vela/server/api/types"
 	"github.com/minio/minio-go/v7"
 )
 
-// Helper methods for uploading objects
-func (c *MinioClient) Upload(ctx context.Context, bucketName, objectName string, data []byte, contentType string) error {
-	c.Logger.Tracef("uploading data to bucket %s", bucketName)
+// Upload uploads an object to a bucket in MinIO.ts
+func (c *MinioClient) Upload(ctx context.Context, object *api.Object) error {
+	c.Logger.Tracef("uploading data to bucket %s", object.ObjectName)
 
-	reader := bytes.NewReader(data)
-	_, err := c.client.PutObject(ctx, bucketName, objectName, reader, int64(len(data)), minio.PutObjectOptions{ContentType: contentType})
+	_, err := c.client.FPutObject(ctx, object.BucketName, object.ObjectName, object.FilePath, minio.PutObjectOptions{})
 	return err
 }

storage/service.go

@@ -13,12 +13,13 @@ type Storage interface {
 	BucketExists(ctx context.Context, bucket *api.Bucket) (bool, error)
 	ListBuckets(ctx context.Context) ([]string, error)
 	// Object Operations
-	Upload(ctx context.Context, bucketName string, objectName string, data []byte, contentType string) error
-	Download(ctx context.Context, bucketName string, objectName string) ([]byte, error)
-	Delete(ctx context.Context, bucketName string, objectName string) error
+	Upload(ctx context.Context, object *api.Object) error
+	Download(ctx context.Context, object *api.Object) error
+	Delete(ctx context.Context, object *api.Object) error
 	ListObjects(ctx context.Context, bucketName string) ([]string, error)
-	//// Presigned URLs
+	// Presigned URLs
 	//GeneratePresignedURL(ctx context.Context, bucket string, key string, expiry int64) (string, error)
+	PresignedGetObject(ctx context.Context, object *api.Object) (string, error)
 	// Object Lifecycle
 	SetBucketLifecycle(ctx context.Context, bucketName *api.Bucket) error
 	GetBucketLifecycle(ctx context.Context, bucket *api.Bucket) (*api.Bucket, error)