Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: add support for seedProjects with descriptions via seedProjectsWithDetails #5864

Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions charts/flyte-binary/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -116,6 +116,8 @@ Chart for basic single Flyte executable deployment
| flyte-core-components.admin.disableClusterResourceManager | bool | `false` | |
| flyte-core-components.admin.disableScheduler | bool | `false` | |
| flyte-core-components.admin.disabled | bool | `false` | |
| flyte-core-components.admin.seedProjectsWithDetails[0].description | string | `"Default project setup."` | |
| flyte-core-components.admin.seedProjectsWithDetails[0].name | string | `"flytesnacks"` | |
| flyte-core-components.admin.seedProjects[0] | string | `"flytesnacks"` | |
| flyte-core-components.dataCatalog.disabled | bool | `false` | |
| flyte-core-components.propeller.disableWebhook | bool | `false` | |
Expand Down
8 changes: 8 additions & 0 deletions charts/flyte-binary/values.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,14 @@ flyte-core-components:
# seedProjects flyte projects to create by default
seedProjects:
- flytesnacks
# seedProjectsWithDetails flyte projects to create by default with description
# If there is an overlap between seedProjects and seedProjectsWithDetails,
# the description provided in seedProjectsWithDetails will take precedence.
# For seedProjects without a corresponding description in seedProjectsWithDetails,
# a default description will be auto-generated for the project.
seedProjectsWithDetails:
- name: flytesnacks
description: Default project setup.
# propeller Configuration to disable propeller or any of its components
propeller:
# disabled Disables flytepropeller
Expand Down
14 changes: 9 additions & 5 deletions cmd/single/config.go
Original file line number Diff line number Diff line change
@@ -1,6 +1,9 @@
package single

import "github.com/flyteorg/flyte/flytestdlib/config"
import (
adminRepositoriesConfig "github.com/flyteorg/flyte/flyteadmin/pkg/repositories/config"
"github.com/flyteorg/flyte/flytestdlib/config"
)

//go:generate pflags Config --default-var=DefaultConfig

Expand All @@ -21,10 +24,11 @@ type Propeller struct {
}

type Admin struct {
Disabled bool `json:"disabled" pflag:",Disables flyteadmin in the single binary mode"`
DisableScheduler bool `json:"disableScheduler" pflag:",Disables Native scheduler in the single binary mode"`
DisableClusterResourceManager bool `json:"disableClusterResourceManager" pflag:",Disables Cluster resource manager"`
SeedProjects []string `json:"seedProjects" pflag:",flyte projects to create by default."`
Disabled bool `json:"disabled" pflag:",Disables flyteadmin in the single binary mode"`
DisableScheduler bool `json:"disableScheduler" pflag:",Disables Native scheduler in the single binary mode"`
DisableClusterResourceManager bool `json:"disableClusterResourceManager" pflag:",Disables Cluster resource manager"`
SeedProjects []string `json:"seedProjects" pflag:",flyte projects to create by default."`
SeedProjectsWithDetails []adminRepositoriesConfig.SeedProject `json:"seedProjectsWithDetails" pflag:",,Detailed configuration for Flyte projects to be created by default."`
}

type DataCatalog struct {
Expand Down
6 changes: 4 additions & 2 deletions cmd/single/start.go
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ import (
datacatalog "github.com/flyteorg/flyte/datacatalog/pkg/rpc/datacatalogservice"
"github.com/flyteorg/flyte/flyteadmin/pkg/clusterresource"
"github.com/flyteorg/flyte/flyteadmin/pkg/common"
adminRepositoriesConfig "github.com/flyteorg/flyte/flyteadmin/pkg/repositories/config"
"github.com/flyteorg/flyte/flyteadmin/pkg/runtime"
adminServer "github.com/flyteorg/flyte/flyteadmin/pkg/server"
"github.com/flyteorg/flyte/flyteadmin/plugins"
Expand Down Expand Up @@ -75,8 +76,9 @@ func startAdmin(ctx context.Context, cfg Admin) error {
if len(cfg.SeedProjects) != 0 {
projects = cfg.SeedProjects
}
logger.Infof(ctx, "Seeding default projects...", projects)
if err := adminServer.SeedProjects(ctx, projects); err != nil {
seedProjects := adminRepositoriesConfig.MergeSeedProjectsWithUniqueNames(projects, cfg.SeedProjectsWithDetails)
logger.Infof(ctx, "Seeding default projects... %v", seedProjects)
if err := adminServer.SeedProjects(ctx, seedProjects); err != nil {
return err
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -92,6 +92,9 @@ data:
disabled: false
seedProjects:
- flytesnacks
seedProjectsWithDetails:
- description: Default project setup.
name: flytesnacks
dataCatalog:
disabled: false
propeller:
Expand Down Expand Up @@ -360,7 +363,7 @@ spec:
app.kubernetes.io/instance: flyte
app.kubernetes.io/component: flyte-binary
annotations:
checksum/configuration: faaefbd3b3b2ddfd4e718bd77c02c632c75e7111dad0a6e25dc415dc88add73f
checksum/configuration: 886440a42b3eeec802cfe60d37885f69e35ffd83e53e625b3c877da5e8c7eb38
checksum/configuration-secret: d5d93f4e67780b21593dc3799f0f6682aab0765e708e4020939975d14d44f929
checksum/cluster-resource-templates: 7dfa59f3d447e9c099b8f8ffad3af466fecbc9cf9f8c97295d9634254a55d4ae
spec:
Expand Down
4 changes: 3 additions & 1 deletion flyteadmin/cmd/entrypoints/migrate.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
"github.com/spf13/cobra"
_ "gorm.io/driver/postgres" // Required to import database driver.

"github.com/flyteorg/flyte/flyteadmin/pkg/repositories/config"
"github.com/flyteorg/flyte/flyteadmin/pkg/server"
)

Expand Down Expand Up @@ -40,7 +41,8 @@
Short: "Seed projects in the database.",
RunE: func(cmd *cobra.Command, args []string) error {
ctx := context.Background()
return server.SeedProjects(ctx, args)
seedProjects := config.UniqueProjectsFromNames(args)
return server.SeedProjects(ctx, seedProjects)

Check warning on line 45 in flyteadmin/cmd/entrypoints/migrate.go

View check run for this annotation

Codecov / codecov/patch

flyteadmin/cmd/entrypoints/migrate.go#L44-L45

Added lines #L44 - L45 were not covered by tests
},
}

Expand Down
55 changes: 50 additions & 5 deletions flyteadmin/pkg/repositories/config/seed_data.go
Original file line number Diff line number Diff line change
Expand Up @@ -10,16 +10,61 @@ import (
"github.com/flyteorg/flyte/flytestdlib/logger"
)

type SeedProject struct {
Name string `json:"name" pflag:",Name of flyte project to create"`
Description string `json:"description" pflag:",Description of flyte project to create"`
}

func UniqueProjectsFromNames(names []string) []SeedProject {
return uniqueProjects(names, nil)
}

// MergeSeedProjectsWithUniqueNames merges seed projects from names and details while maintaining uniqueness
func MergeSeedProjectsWithUniqueNames(seedProjects []string, seedProjectsWithDetails []SeedProject) []SeedProject {
return uniqueProjects(seedProjects, seedProjectsWithDetails)
}

func uniqueProjects(seedProjects []string, seedProjectsWithDetails []SeedProject) []SeedProject {
// Track unique project names
seen := make(map[string]struct{})

// Create the final result slice
var combinedProjects []SeedProject

// First, add all projects from SeedProjectsWithDetails to the map
for _, project := range seedProjectsWithDetails {
// Handle the duplication
if _, exists := seen[project.Name]; !exists {
seen[project.Name] = struct{}{}
combinedProjects = append(combinedProjects, project)
}
}

// Process SeedProjects
for _, projectName := range seedProjects {
// Check if project not exists in SeedProjectsWithDetails
if _, exists := seen[projectName]; !exists {
seen[projectName] = struct{}{}
combinedProjects = append(combinedProjects, SeedProject{
Name: projectName,
Description: fmt.Sprintf("%s description", projectName),
})
}
}

return combinedProjects
}

// Returns a function to seed the database with default values.
func SeedProjects(db *gorm.DB, projects []string) error {
func SeedProjects(db *gorm.DB, projects []SeedProject) error {
tx := db.Begin()
for _, project := range projects {
projectModel := models.Project{
Identifier: project,
Name: project,
Description: fmt.Sprintf("%s description", project),
Identifier: project.Name,
Name: project.Name,
Description: project.Description,
}
if err := tx.Where(models.Project{Identifier: project}).Omit("id").FirstOrCreate(&projectModel).Error; err != nil {
if err := tx.Where(models.Project{Identifier: project.Name}).Omit("id").FirstOrCreate(&projectModel).Error; err != nil {
logger.Warningf(context.Background(), "failed to save project [%s]", project)
tx.Rollback()
return err
Expand Down
Loading
Loading