diff --git a/db/db.go b/db/db.go index ed39a0db6..4983208cf 100644 --- a/db/db.go +++ b/db/db.go @@ -22,7 +22,6 @@ import ( "strings" "github.com/memphisdev/memphis/conf" - "github.com/memphisdev/memphis/models" "context" @@ -597,7 +596,7 @@ func createTables(MetadataDbClient MetadataStorage) error { db := MetadataDbClient.Client ctx := MetadataDbClient.Ctx - tables := []string{alterTenantsTable, tenantsTable, alterUsersTable, usersTable, alterAuditLogsTable, auditLogsTable, alterConfigurationsTable, configurationsTable, alterIntegrationsTable, integrationsTable, alterSchemasTable, schemasTable, alterTagsTable, tagsTable, alterStationsTable, stationsTable, alterDlsMsgsTable, dlsMessagesTable, alterConsumersTable, consumersTable, alterSchemaVerseTable, schemaVersionsTable, alterProducersTable, producersTable, alterConnectionsTable, asyncTasksTable, alterAsyncTasks, testEventsTable, installedFunctionsTable, attachedFunctionsTable} + tables := []string{alterTenantsTable, tenantsTable, alterUsersTable, usersTable, alterAuditLogsTable, auditLogsTable, alterConfigurationsTable, configurationsTable, alterIntegrationsTable, integrationsTable, alterSchemasTable, schemasTable, alterTagsTable, tagsTable, alterStationsTable, stationsTable, alterDlsMsgsTable, dlsMessagesTable, alterConsumersTable, consumersTable, alterSchemaVerseTable, schemaVersionsTable, alterProducersTable, producersTable, alterConnectionsTable, asyncTasksTable, alterAsyncTasks, testEventsTable, functionsTable, attachedFunctionsTable} for _, table := range tables { _, err := db.Exec(ctx, table) diff --git a/db/db_cloud.go b/db/db_cloud.go index 34d289cf2..d222c9fb9 100644 --- a/db/db_cloud.go +++ b/db/db_cloud.go @@ -12,7 +12,7 @@ package db const testEventsTable = `` -const installedFunctionsTable = `` +const functionsTable = `` const attachedFunctionsTable = `` type FunctionSchema struct { diff --git a/models/functions.go b/models/functions.go index eb5dfe548..ba3d2971d 100644 --- a/models/functions.go +++ b/models/functions.go @@ -13,29 +13,65 @@ package models import "time" -type FunctionsResult struct { - FunctionName string `json:"function_name"` - Description string `json:"description"` - Tags []string `json:"tags"` - RunTime string `json:"runtime"` - Memory int `json:"memory"` - Storgae int `json:"storgae"` - LastCommit time.Time `json:"last_commit"` - Link *string `json:"link"` - Repository string `json:"repository"` - Branch string `json:"branch"` - Owner string `json:"owner"` - EnvironmentVars map[string]string `json:"environment_vars"` - ScmType string `json:"scm_type"` - Language string `json:"language"` - InstallInProgress bool `json:"install_in_progress"` - UpdatesAvailable bool `json:"updates_available"` - ByMemphis bool `json:"by_memphis"` +type Function struct { + ID int `json:"id"` + FunctionName string `json:"function_name"` + Description string `json:"description"` + Tags []string `json:"tags"` + Runtime string `json:"runtime"` + Dependencies string `json:"dependencies"` + EnvironmentVars []map[string]interface{} `json:"environment_vars"` + Memory int `json:"memory"` + Storage int `json:"storage"` + Handler string `json:"handler"` + TenantName string `json:"tenant_name"` + Scm string `json:"scm"` + Owner string `json:"owner"` + Repo string `json:"repo"` + Branch string `json:"branch"` + UpdatedAt time.Time `json:"installed_updated_at"` + Version int `json:"installed_version"` + InProgress bool `json:"installed_in_progress"` + ComputeEngine string `json:"compute_engine"` + Installed bool `json:"installed"` + IsValid bool `json:"is_valid"` + InvalidReason string `json:"invalid_reason"` + UpdatesAvailable bool `json:"updates_available"` + ByMemphis bool `json:"by_memphis"` } +type FunctionResult struct { + ID int `json:"id"` + FunctionName string `json:"function_name"` + Description string `json:"description"` + Tags []string `json:"tags"` + Runtime string `json:"runtime"` + Dependencies string `json:"dependencies"` + EnvironmentVars []map[string]interface{} `json:"environment_vars"` + Memory int `json:"memory"` + Storage int `json:"storage"` + Handler string `json:"handler"` + TenantName string `json:"tenant_name"` + Scm string `json:"scm"` + Owner string `json:"owner"` + Repo string `json:"repo"` + Branch string `json:"branch"` + UpdatedAt time.Time `json:"installed_updated_at"` + Version int `json:"installed_version"` + InProgress bool `json:"installed_in_progress"` + ComputeEngine string `json:"compute_engine"` + Installed bool `json:"installed"` + IsValid bool `json:"is_valid"` + InvalidReason string `json:"invalid_reason"` + UpdatesAvailable bool `json:"updates_available"` + ByMemphis bool `json:"by_memphis"` + Language string `json:"language"` + Link *string `json:"link,omitempty"` + LastCommit *time.Time `json:"last_commit,omitempty"` +} type FunctionsRes struct { - InstalledFunctions []FunctionsResult `json:"installed_functions"` - OtherFunctions []FunctionsResult `json:"other_functions"` + InstalledFunctions []FunctionResult `json:"installed_functions"` + OtherFunctions []FunctionResult `json:"other_functions"` ScmIntegrated bool `json:"scm_integrated"` ConnectedRepos []map[string]interface{} `json:"connected_repos"` } diff --git a/server/memphis_handlers_functions_cloud.go b/server/memphis_handlers_functions_cloud.go index 165689bc8..869b2e921 100644 --- a/server/memphis_handlers_functions_cloud.go +++ b/server/memphis_handlers_functions_cloud.go @@ -17,6 +17,7 @@ import ( "fmt" "net/http" "regexp" + "time" "strings" @@ -66,15 +67,53 @@ func (fh FunctionsHandler) GetFunctions(tenantName string) (models.FunctionsRes, installedFunctions := functions["installed"] OtherFunctions := functions["other"] if len(installedFunctions) == 0 { - installedFunctions = []models.FunctionsResult{} + installedFunctions = []models.FunctionResult{} } if len(OtherFunctions) == 0 { - OtherFunctions = []models.FunctionsResult{} + OtherFunctions = []models.FunctionResult{} + } + + var lastModified *time.Time + OtherFunctions = []models.FunctionResult{} + for _, function := range functions["other"] { + if function.Owner == memphisDevFunctionsOwnerName && function.Repo == memphisDevFunctionsRepoName { + otherFunctionResult := models.FunctionResult{ + FunctionName: function.FunctionName, + Description: function.Description, + Tags: function.Tags, + Runtime: function.Runtime, + Dependencies: function.Dependencies, + EnvironmentVars: function.EnvironmentVars, + Memory: function.Memory, + Storage: function.Storage, + Handler: function.Handler, + Scm: "github", + Repo: function.Repo, + Branch: function.Branch, + Owner: function.Owner, + Language: function.Language, + Version: -1, + IsValid: function.IsValid, + InvalidReason: function.InvalidReason, + InProgress: false, + UpdatesAvailable: false, + ByMemphis: function.ByMemphis, + TenantName: function.TenantName, + } + OtherFunctions = append(OtherFunctions, otherFunctionResult) + lastModified = function.LastCommit + } } memphisDevFucntions := []map[string]interface{}{} - memphisDevFucntions = append(memphisDevFucntions, memphisFunctions) + memphisFunc := map[string]interface{}{ + "repo_name": memphisFunctions["repo_name"].(string), + "branch": memphisFunctions["branch"].(string), + "owner": memphisFunctions["repo_owner"].(string), + "last_modified": lastModified, + } + memphisDevFucntions = append(memphisDevFucntions, memphisFunc) allFunctions := models.FunctionsRes{ InstalledFunctions: installedFunctions, @@ -220,8 +259,8 @@ func getRepoContent(url, accessToken string, body models.GetFunctionDetails) (in return response, nil } -func GetFunctionsDetails(functionsDetails map[string][]functionDetails) (map[string][]models.FunctionsResult, error) { - functions := map[string][]models.FunctionsResult{} +func GetFunctionsDetails(functionsDetails map[string][]functionDetails) (map[string][]models.FunctionResult, error) { + functions := map[string][]models.FunctionResult{} for key, functionDetails := range functionsDetails { for _, funcDetailsPerInstalled := range functionDetails { fucntionContentMap := funcDetailsPerInstalled.ContentMap @@ -230,6 +269,9 @@ func GetFunctionsDetails(functionsDetails map[string][]functionDetails) (map[str repo := funcDetailsPerInstalled.RepoName branch := funcDetailsPerInstalled.Branch owner := funcDetailsPerInstalled.Owner + tenantName := funcDetailsPerInstalled.TenantName + isValid := funcDetailsPerInstalled.IsValid + invalidReason := funcDetailsPerInstalled.InvalidReason tagsInterfaceSlice, ok := fucntionContentMap["tags"].([]interface{}) tagsStrings := []string{} if ok { @@ -244,17 +286,20 @@ func GetFunctionsDetails(functionsDetails map[string][]functionDetails) (map[str } } - var environmentVarsStrings map[string]string + environmentVarsStrings := []map[string]interface{}{} environmentVarsInterfaceSlice, ok := fucntionContentMap["environment_vars"].([]interface{}) if ok { - environmentVarsStrings = make(map[string]string, len(fucntionContentMap["environment_vars"].([]interface{}))) - for _, environmentVar := range environmentVarsInterfaceSlice { - environmentVarMap := environmentVar.(map[interface{}]interface{}) + for _, environmentVarInterface := range environmentVarsInterfaceSlice { + environmentVarMap, _ := environmentVarInterface.(map[interface{}]interface{}) + environmentVar := make(map[string]interface{}) for k, v := range environmentVarMap { - if str, ok := v.(string); ok { - environmentVarsStrings[k.(string)] = str + if key, ok := k.(string); ok { + if val, ok := v.(string); ok { + environmentVar[key] = val + } } } + environmentVarsStrings = append(environmentVarsStrings, environmentVar) } } description, ok := fucntionContentMap["description"].(string) @@ -262,12 +307,15 @@ func GetFunctionsDetails(functionsDetails map[string][]functionDetails) (map[str description = "" } - runtime := fucntionContentMap["runtime"].(string) - regex := regexp.MustCompile(`[0-9]+|\\.$`) - language := regex.ReplaceAllString(runtime, "") - language = strings.TrimRight(language, ".") - if strings.Contains(language, "-edge") { - language = strings.Trim(language, ".-edge") + runtime, ok := fucntionContentMap["runtime"].(string) + var language string + if ok { + regex := regexp.MustCompile(`[0-9]+|\\.$`) + language = regex.ReplaceAllString(runtime, "") + language = strings.TrimRight(language, ".") + if strings.Contains(language, "-edge") { + language = strings.Trim(language, ".-edge") + } } byMemphis := false @@ -275,24 +323,38 @@ func GetFunctionsDetails(functionsDetails map[string][]functionDetails) (map[str byMemphis = true } - functionDetails := models.FunctionsResult{ - FunctionName: fucntionContentMap["function_name"].(string), - Description: description, - Tags: tagsStrings, - RunTime: runtime, - LastCommit: *commit.Commit.Committer.Date, - Link: link, - Repository: repo, - Branch: branch, - Owner: owner, - Memory: fucntionContentMap["memory"].(int), - Storgae: fucntionContentMap["storage"].(int), - EnvironmentVars: environmentVarsStrings, - Language: language, - ScmType: "github", - InstallInProgress: false, - UpdatesAvailable: false, - ByMemphis: byMemphis, + handler := "" + if _, ok := fucntionContentMap["handler"].(string); ok { + handler = fucntionContentMap["handler"].(string) + } + var lastCommit *time.Time + if commit != nil { + lastCommit = &*commit.Commit.Committer.Date + } + + functionDetails := models.FunctionResult{ + FunctionName: fucntionContentMap["function_name"].(string), + Description: description, + Tags: tagsStrings, + Runtime: runtime, + Dependencies: fucntionContentMap["dependencies"].(string), + EnvironmentVars: environmentVarsStrings, + Memory: fucntionContentMap["memory"].(int), + Storage: fucntionContentMap["storage"].(int), + Handler: handler, + Scm: "github", + Repo: repo, + Branch: branch, + Owner: owner, + LastCommit: lastCommit, + Link: link, + Language: language, + InProgress: false, + UpdatesAvailable: false, + ByMemphis: byMemphis, + TenantName: tenantName, + IsValid: isValid, + InvalidReason: invalidReason, } functions[key] = append(functions[key], functionDetails) diff --git a/server/source_code_management_cloud.go b/server/source_code_management_cloud.go index 41f25a6c5..b65a80447 100644 --- a/server/source_code_management_cloud.go +++ b/server/source_code_management_cloud.go @@ -22,22 +22,25 @@ type GetSourceCodeBranchesSchema struct { } type functionDetails struct { - DirectoryUrl *string `json:"directory_content"` - Commit *github.RepositoryCommit `json:"commit"` - ContentMap map[string]interface{} `json:"content_map"` - RepoName string `json:"repo_name"` - Branch string `json:"branch"` - Scm string `json:"scm"` - Owner string `json:"owner"` + DirectoryUrl *string `json:"directory_content"` + Commit *github.RepositoryCommit `json:"commit"` + ContentMap map[string]interface{} `json:"content_map"` + RepoName string `json:"repo_name"` + Branch string `json:"branch"` + Scm string `json:"scm"` + Owner string `json:"owner"` + TenantName string `json:"tenant_name"` + IsValid bool `json:"is_valid"` + InvalidReason string `json:"invalid_reason"` } func getSourceCodeDetails(tenantName string, getAllReposSchema interface{}, actionType string) (models.Integration, interface{}, error) { return models.Integration{}, map[string]string{}, nil } -func GetContentOfSelectedRepo(connectedRepo map[string]interface{}, contentDetails map[string][]functionDetails) (map[string][]functionDetails, error) { +func GetContentOfSelectedRepo(connectedRepo map[string]interface{}, contentDetails map[string][]functionDetails, tenantName string) (map[string][]functionDetails, error) { var err error - contentDetails, err = GetGithubContentFromConnectedRepo(connectedRepo, contentDetails) + contentDetails, err = GetGithubContentFromConnectedRepo(connectedRepo, contentDetails, tenantName) if err != nil { return contentDetails, err } @@ -62,7 +65,7 @@ func GetContentOfSelectedRepos(tenantName string) (map[string][]functionDetails, for _, connectedRepoPerIntegration := range connectedRepos { for _, connectedRepo := range connectedRepoPerIntegration { connectedRepoRes := connectedRepo.(map[string]interface{}) - contentDetails, err = GetContentOfSelectedRepo(connectedRepoRes, contentDetails) + contentDetails, err = GetContentOfSelectedRepo(connectedRepoRes, contentDetails, tenantName) if err != nil { return contentDetails, scmIntegrated, err } diff --git a/server/source_code_management_github_cloud.go b/server/source_code_management_github_cloud.go index 64ba082d0..278df911c 100644 --- a/server/source_code_management_github_cloud.go +++ b/server/source_code_management_github_cloud.go @@ -3,6 +3,8 @@ package server import ( "context" "encoding/base64" + "fmt" + "strings" "github.com/memphisdev/memphis/models" "gopkg.in/yaml.v2" @@ -73,7 +75,7 @@ func containsElement(arr []string, val string) bool { return false } -func GetGithubContentFromConnectedRepo(connectedRepo map[string]interface{}, functionsDetails map[string][]functionDetails) (map[string][]functionDetails, error) { +func GetGithubContentFromConnectedRepo(connectedRepo map[string]interface{}, functionsDetails map[string][]functionDetails, tenantName string) (map[string][]functionDetails, error) { branch := connectedRepo["branch"].(string) repo := connectedRepo["repo_name"].(string) owner := connectedRepo["repo_owner"].(string) @@ -130,52 +132,74 @@ func GetGithubContentFromConnectedRepo(connectedRepo map[string]interface{}, fun contentMap["storage"] = int64(512) * 1024 * 1024 } - // TODO: need to add according to the cloud changes - // if contentMap["dependencies"].(string) == "" { - // switch contentMap["language"] { - // case "go": - // contentMap["dependencies"] = "go.mod" - // case "nodejs": - // contentMap["dependencies"] = "package.json" - // case "python": - // contentMap["dependencies"] = "req.txt" - // } - // } - - // splitPath := strings.Split(*fileContent.Path, "/") - // path := strings.TrimSpace(splitPath[0]) - // if path != contentMap["function_name"].(string) { - // // errMsg := fmt.Sprintf("In the repository %s, there was an incompatibility between the function name in the git %s and the function name in the YAML file %s", repo, splitPath[0], contentMap["function_name"].(string)) - // continue - // } - // if strings.Contains(path, "") { - // // errMsg := fmt.Sprintf("In the repository %s, the function name in the yaml %s can't contains spaces", repo, contentMap["function_name"].(string)) - // continue - // } - // + if contentMap["dependencies"].(string) == "" { + switch contentMap["language"] { + case "go": + contentMap["dependencies"] = "go.mod" + case "nodejs": + contentMap["dependencies"] = "package.json" + case "python": + contentMap["dependencies"] = "requirements.txt" + } + } + + splitPath := strings.Split(*fileContent.Path, "/") + path := strings.TrimSpace(splitPath[0]) err = validateYamlContent(contentMap) if err != nil { isValidFileYaml = false + fileDetails := functionDetails{ + ContentMap: contentMap, + RepoName: repo, + Branch: branch, + Owner: owner, + DirectoryUrl: directoryContent.HTMLURL, + TenantName: tenantName, + } + message := fmt.Sprintf("In the repository %s, the yaml file %s is invalid: %s", repo, splitPath[0], err.Error()) + serv.Warnf("[tenant: %s]GetGithubContentFromConnectedRepo: %s", tenantName, message) + fileDetails.IsValid = false + fileDetails.InvalidReason = message + functionsDetails["other"] = append(functionsDetails["other"], fileDetails) continue } isValidFileYaml = true - commit, _, err = client.Repositories.GetCommit(context.Background(), owner, repo, branch) if err != nil { continue } + fileDetails := functionDetails{ + Commit: commit, + ContentMap: contentMap, + RepoName: repo, + Branch: branch, + Owner: owner, + DirectoryUrl: directoryContent.HTMLURL, + TenantName: tenantName, + } + + if path != contentMap["function_name"].(string) { + message := fmt.Sprintf("In the repository %s, function name %s in git doesn't match the function_name field %s in YAML file.", repo, splitPath[0], contentMap["function_name"].(string)) + serv.Warnf("[tenant: %s]GetGithubContentFromConnectedRepo: %s", tenantName, message) + fileDetails.IsValid = false + fileDetails.InvalidReason = message + functionsDetails["other"] = append(functionsDetails["other"], fileDetails) + continue + } + if strings.Contains(path, " ") { + message := fmt.Sprintf("In the repository %s, the function name %s in the YAML file cannot contain spaces", repo, contentMap["function_name"].(string)) + serv.Warnf("[tenant: %s]GetGithubContentFromConnectedRepo: %s", tenantName, message) + fileDetails.IsValid = false + fileDetails.InvalidReason = message + functionsDetails["other"] = append(functionsDetails["other"], fileDetails) + continue + } + if isValidFileYaml { countFunctions++ - fileDetails := functionDetails{ - Commit: commit, - ContentMap: contentMap, - RepoName: repo, - Branch: branch, - Owner: owner, - DirectoryUrl: directoryContent.HTMLURL, - } + fileDetails.IsValid = true functionsDetails["other"] = append(functionsDetails["other"], fileDetails) break }