From a1afb99d2538fd0f052e738f257d2d727eaac4f5 Mon Sep 17 00:00:00 2001 From: davidvader Date: Wed, 29 Dec 2021 15:24:12 -0600 Subject: [PATCH 01/36] feat: visualization WIP --- api/dag.go | 268 ++++++++++++++++++++++++++++++++++++++++++++++++ router/build.go | 1 + 2 files changed, 269 insertions(+) create mode 100644 api/dag.go diff --git a/api/dag.go b/api/dag.go new file mode 100644 index 000000000..cf177d356 --- /dev/null +++ b/api/dag.go @@ -0,0 +1,268 @@ +// Copyright (c) 2021 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package api + +import ( + "fmt" + "net/http" + "strconv" + "strings" + + "github.com/go-vela/server/compiler" + "github.com/go-vela/server/database" + "github.com/go-vela/server/router/middleware/org" + "github.com/go-vela/server/router/middleware/user" + "github.com/go-vela/server/scm" + "github.com/go-vela/types" + "github.com/go-vela/types/constants" + "github.com/go-vela/types/library" + "github.com/go-vela/types/pipeline" + + "github.com/go-vela/server/router/middleware/build" + "github.com/go-vela/server/router/middleware/repo" + "github.com/go-vela/server/util" + + "github.com/gin-gonic/gin" + "github.com/sirupsen/logrus" +) + +// swagger:operation GET /api/v1/repos/{org}/{repo}/builds/{build}/dag builds GetBuildDAG +// +// Get directed a-cyclical graph for a build in the configured backend +// +// --- +// produces: +// - application/json +// parameters: +// - in: path +// name: org +// description: Name of the org +// required: true +// type: string +// - in: path +// name: repo +// description: Name of the repo +// required: true +// type: string +// - in: path +// name: build +// description: Build number +// required: true +// type: integer +// security: +// - ApiKeyAuth: [] +// responses: +// '200': +// description: Successfully retrieved dag for the build +// schema: +// type: array +// items: +// "$ref": "#/definitions/DAG" +// '500': +// description: Unable to retrieve dag for the build +// schema: +// "$ref": "#/definitions/Error" + +// GetBuildDAG represents the API handler to capture a +// directed a-cyclical graph for a build from the configured backend. +func GetBuildDAG(c *gin.Context) { + // capture middleware values + b := build.Retrieve(c) + o := org.Retrieve(c) + r := repo.Retrieve(c) + u := user.Retrieve(c) + m := c.MustGet("metadata").(*types.Metadata) + + entry := fmt.Sprintf("%s/%d", r.GetFullName(), b.GetNumber()) + + // update engine logger with API metadata + // + // https://pkg.go.dev/github.com/sirupsen/logrus?tab=doc#Entry.WithFields + logrus.WithFields(logrus.Fields{ + "build": b.GetNumber(), + "org": o, + "repo": r.GetName(), + "user": u.GetName(), + }).Infof("getting all steps for build %s", entry) + + // retrieve the steps for the build from the step table + steps := []*library.Step{} + page := 1 + perPage := 100 + for page > 0 { + // retrieve build steps (per page) from the database + stepsPart, err := database.FromContext(c).GetBuildStepList(b, page, perPage) + if err != nil { + retErr := fmt.Errorf("unable to retrieve steps for build %s: %w", entry, err) + util.HandleError(c, http.StatusNotFound, retErr) + return + } + + // add page of steps to list steps + steps = append(steps, stepsPart...) + + // assume no more pages exist if under 100 results are returned + // + // nolint: gomnd // ignore magic number + if len(stepsPart) < 100 { + page = 0 + } else { + page++ + } + } + + if len(steps) == 0 { + retErr := fmt.Errorf("no steps found for build %s", entry) + util.HandleError(c, http.StatusNotFound, retErr) + return + } + + logrus.Info("retrieving pipeline configuration file") + + // send API call to capture the pipeline configuration file + config, err := scm.FromContext(c).ConfigBackoff(u, r, b.GetCommit()) + if err != nil { + // nolint: lll // ignore long line length due to error message + retErr := fmt.Errorf("%s: failed to get pipeline configuration for %s: %v", baseErr, r.GetFullName(), err) + util.HandleError(c, http.StatusNotFound, retErr) + + return + } + + // variable to store changeset files + var files []string + // check if the build event is not issue_comment + if !strings.EqualFold(b.GetEvent(), constants.EventComment) { + // check if the build event is not pull_request + if !strings.EqualFold(b.GetEvent(), constants.EventPull) { + // send API call to capture list of files changed for the commit + files, err = scm.FromContext(c).Changeset(u, r, b.GetCommit()) + if err != nil { + retErr := fmt.Errorf("%s: failed to get changeset for %s: %v", baseErr, r.GetFullName(), err) + util.HandleError(c, http.StatusInternalServerError, retErr) + + return + } + } + } + + logrus.Info("compiling pipeline") + // parse and compile the pipeline configuration file + p, err := compiler.FromContext(c). + Duplicate(). + WithBuild(b). + WithFiles(files). + WithMetadata(m). + WithRepo(r). + WithUser(u). + Compile(config) + if err != nil { + // format the error message with extra information + err = fmt.Errorf("unable to compile pipeline configuration for %s: %v", r.GetFullName(), err) + + // log the error for traceability + logrus.Error(err.Error()) + + retErr := fmt.Errorf("%s: %v", baseErr, err) + util.HandleError(c, http.StatusInternalServerError, retErr) + + return + } + + // skip the build if only the init or clone steps are found + skip := skipEmptyBuild(p) + if skip != "" { + c.JSON(http.StatusOK, skip) + return + } + + logrus.Info("creating dag using 'needs'") + + // group library steps by stage name + stages := map[string][]*library.Step{} + + for _, _step := range steps { + if _, ok := stages[_step.GetStage()]; !ok { + stages[_step.GetStage()] = []*library.Step{} + } + stages[_step.GetStage()] = append(stages[_step.GetStage()], _step) + } + + // create nodes from pipeline stages + nodes := []*Node{} + + for _, stage := range p.Stages { + for _, step := range stage.Steps { + step.Environment = nil + } + nodeID := (len(nodes)) + node := Node{ + Label: stage.Name, + Stage: stage, + Steps: stages[stage.Name], + + ID: strconv.Itoa(nodeID), + } + nodes = append(nodes, &node) + } + + // create edges from nodes + // edges := []*Edge{} + + // loop over nodes + for _, destinationNode := range nodes { + // compare all nodes against all nodes + for _, sourceNode := range nodes { + // dont compare the same node + if destinationNode.ID != sourceNode.ID { + + // check destination node needs + for _, need := range (*destinationNode.Stage).Needs { + // check if destination needs source stage + if sourceNode.Stage.Name == need { + // a node is represented by a destination stage that + // requires source stage(s) + // edgeID := (len(edges)) + // edge := Edge{ + // EdgeID: edgeID, + // NodeID: sourceNode.NodeID, + // } + + // collect edge to increment edge_id + // edges = append(edges, &edge) + + // add the edge to the node + if (*destinationNode).ParentIDs == nil { + (*destinationNode).ParentIDs = make([]string, 0) + } + (*destinationNode).ParentIDs = append((*destinationNode).ParentIDs, sourceNode.ID) + } + } + + } + } + } + + c.JSON(http.StatusOK, nodes) +} + +type DAG struct { + Nodes []*Node `json:"nodes"` +} + +type Node struct { + Label string `json:"label"` + Stage *pipeline.Stage `json:"stage"` + Steps []*library.Step `json:"steps"` + + // d3 stuff + ID string `json:"id"` + ParentIDs []string `json:"parent_ids"` +} + +type Edge struct { + EdgeID int `json:"edge_id"` + NodeID int `json:"node_id"` +} diff --git a/router/build.go b/router/build.go index c12350260..576921e80 100644 --- a/router/build.go +++ b/router/build.go @@ -58,6 +58,7 @@ func BuildHandlers(base *gin.RouterGroup) { build.DELETE("", perm.MustPlatformAdmin(), api.DeleteBuild) build.DELETE("/cancel", executors.Establish(), perm.MustWrite(), api.CancelBuild) build.GET("/logs", perm.MustRead(), api.GetBuildLogs) + build.GET("/dag", perm.MustRead(), api.GetBuildDAG) // Service endpoints // * Log endpoints From 6be0aef0e0d81d8faba2899a1ad0ac67e45263dd Mon Sep 17 00:00:00 2001 From: davidvader Date: Wed, 12 Jan 2022 16:51:53 -0600 Subject: [PATCH 02/36] wip: arquint layout DEPRECATED --- api/dag.go | 64 +++++++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 56 insertions(+), 8 deletions(-) diff --git a/api/dag.go b/api/dag.go index cf177d356..80c255d92 100644 --- a/api/dag.go +++ b/api/dag.go @@ -191,37 +191,37 @@ func GetBuildDAG(c *gin.Context) { } // create nodes from pipeline stages - nodes := []*Node{} + nodes := make(map[string]*Node) for _, stage := range p.Stages { for _, step := range stage.Steps { step.Environment = nil } - nodeID := (len(nodes)) + nodeID := strconv.Itoa(len(nodes)) node := Node{ Label: stage.Name, Stage: stage, Steps: stages[stage.Name], - ID: strconv.Itoa(nodeID), + ID: nodeID, } - nodes = append(nodes, &node) + nodes[nodeID] = &node } // create edges from nodes // edges := []*Edge{} - + links := [][]string{} // loop over nodes for _, destinationNode := range nodes { // compare all nodes against all nodes for _, sourceNode := range nodes { // dont compare the same node if destinationNode.ID != sourceNode.ID { - // check destination node needs for _, need := range (*destinationNode.Stage).Needs { // check if destination needs source stage if sourceNode.Stage.Name == need { + links = append(links, []string{sourceNode.ID, destinationNode.ID}) // a node is represented by a destination stage that // requires source stage(s) // edgeID := (len(edges)) @@ -245,11 +245,59 @@ func GetBuildDAG(c *gin.Context) { } } - c.JSON(http.StatusOK, nodes) + roots := []string{} + for _, node := range nodes { + if (*node).ParentIDs == nil { + logrus.Errorf("no parentIDs for node: %v", node.ID) + roots = append(roots, node.ID) + } + } + logrus.Infof("roots: %v", roots) + + dag := DAG{ + Nodes: nodes, + Links: links, + } + + // g := graphviz.New() + // graph, err := g.Graph() + // if err != nil { + // log.Fatal(err) + // } + // defer func() { + // if err := graph.Close(); err != nil { + // log.Fatal(err) + // } + // g.Close() + // }() + + // nodeA, err := graph.CreateNode("a") + // if err != nil { + // log.Fatal(err) + // } + // nodeB, err := graph.CreateNode("b") + // if err != nil { + // log.Fatal(err) + // } + // e, err := graph.CreateEdge("e", nodeA, nodeB) + // if err != nil { + // log.Fatal(err) + // } + // e.SetLabel("edgeE") + // var buf bytes.Buffer + // if err := g.Render(graph, "dot", &buf); err != nil { + // log.Fatal(err) + // } + + // // print + // fmt.Println(buf.String()) + + c.JSON(http.StatusOK, dag) } type DAG struct { - Nodes []*Node `json:"nodes"` + Nodes map[string]*Node `json:"nodes"` + Links [][]string `json:"links"` } type Node struct { From 83fca69fece8fa2285f76cad01ed713adf37ed09 Mon Sep 17 00:00:00 2001 From: davidvader Date: Sat, 15 Jan 2022 15:40:57 -0600 Subject: [PATCH 03/36] chore: clean up and convert to DOT edges format --- api/{dag.go => graph.go} | 138 ++++++++++++--------------------------- router/build.go | 2 +- 2 files changed, 43 insertions(+), 97 deletions(-) rename api/{dag.go => graph.go} (72%) diff --git a/api/dag.go b/api/graph.go similarity index 72% rename from api/dag.go rename to api/graph.go index 80c255d92..051791e43 100644 --- a/api/dag.go +++ b/api/graph.go @@ -7,7 +7,6 @@ package api import ( "fmt" "net/http" - "strconv" "strings" "github.com/go-vela/server/compiler" @@ -28,7 +27,28 @@ import ( "github.com/sirupsen/logrus" ) -// swagger:operation GET /api/v1/repos/{org}/{repo}/builds/{build}/dag builds GetBuildDAG +// graph contains nodes, and relationships between nodes, or edges. +// a node is a pipeline stage and its relevant steps. +// an edge is a relationship between nodes, defined by the 'needs' tag. +type graph struct { + Nodes map[int]*node `json:"nodes"` + Edges []*edge `json:"edges"` +} + +// node represents is a pipeline stage and its relevant steps. +type node struct { + Label string `json:"label"` + Stage *pipeline.Stage `json:"stage"` + Steps []*library.Step `json:"steps"` + ID int `json:"id"` +} + +type edge struct { + Source int `json:"source"` + Destination int `json:"destination"` +} + +// swagger:operation GET /api/v1/repos/{org}/{repo}/builds/{build}/graph builds GetBuildGraph // // Get directed a-cyclical graph for a build in the configured backend // @@ -55,19 +75,19 @@ import ( // - ApiKeyAuth: [] // responses: // '200': -// description: Successfully retrieved dag for the build +// description: Successfully retrieved graph for the build // schema: // type: array // items: -// "$ref": "#/definitions/DAG" +// "$ref": "#/definitions/Graph" // '500': -// description: Unable to retrieve dag for the build +// description: Unable to retrieve graph for the build // schema: // "$ref": "#/definitions/Error" -// GetBuildDAG represents the API handler to capture a +// GetBuildGraph represents the API handler to capture a // directed a-cyclical graph for a build from the configured backend. -func GetBuildDAG(c *gin.Context) { +func GetBuildGraph(c *gin.Context) { // capture middleware values b := build.Retrieve(c) o := org.Retrieve(c) @@ -182,7 +202,6 @@ func GetBuildDAG(c *gin.Context) { // group library steps by stage name stages := map[string][]*library.Step{} - for _, _step := range steps { if _, ok := stages[_step.GetStage()]; !ok { stages[_step.GetStage()] = []*library.Step{} @@ -191,26 +210,26 @@ func GetBuildDAG(c *gin.Context) { } // create nodes from pipeline stages - nodes := make(map[string]*Node) - + nodes := make(map[int]*node) for _, stage := range p.Stages { + // scrub the environment for _, step := range stage.Steps { step.Environment = nil } - nodeID := strconv.Itoa(len(nodes)) - node := Node{ + nodeID := len(nodes) + node := node{ Label: stage.Name, Stage: stage, Steps: stages[stage.Name], - - ID: nodeID, + ID: nodeID, } nodes[nodeID] = &node } // create edges from nodes - // edges := []*Edge{} - links := [][]string{} + // an edge is as a relationship between two nodes + // that is defined by the 'needs' tag + edges := []*edge{} // loop over nodes for _, destinationNode := range nodes { // compare all nodes against all nodes @@ -221,23 +240,11 @@ func GetBuildDAG(c *gin.Context) { for _, need := range (*destinationNode.Stage).Needs { // check if destination needs source stage if sourceNode.Stage.Name == need { - links = append(links, []string{sourceNode.ID, destinationNode.ID}) - // a node is represented by a destination stage that - // requires source stage(s) - // edgeID := (len(edges)) - // edge := Edge{ - // EdgeID: edgeID, - // NodeID: sourceNode.NodeID, - // } - - // collect edge to increment edge_id - // edges = append(edges, &edge) - - // add the edge to the node - if (*destinationNode).ParentIDs == nil { - (*destinationNode).ParentIDs = make([]string, 0) + edge := &edge{ + Source: sourceNode.ID, + Destination: destinationNode.ID, } - (*destinationNode).ParentIDs = append((*destinationNode).ParentIDs, sourceNode.ID) + edges = append(edges, edge) } } @@ -245,72 +252,11 @@ func GetBuildDAG(c *gin.Context) { } } - roots := []string{} - for _, node := range nodes { - if (*node).ParentIDs == nil { - logrus.Errorf("no parentIDs for node: %v", node.ID) - roots = append(roots, node.ID) - } - } - logrus.Infof("roots: %v", roots) - - dag := DAG{ + // construct the response + dag := graph{ Nodes: nodes, - Links: links, + Edges: edges, } - // g := graphviz.New() - // graph, err := g.Graph() - // if err != nil { - // log.Fatal(err) - // } - // defer func() { - // if err := graph.Close(); err != nil { - // log.Fatal(err) - // } - // g.Close() - // }() - - // nodeA, err := graph.CreateNode("a") - // if err != nil { - // log.Fatal(err) - // } - // nodeB, err := graph.CreateNode("b") - // if err != nil { - // log.Fatal(err) - // } - // e, err := graph.CreateEdge("e", nodeA, nodeB) - // if err != nil { - // log.Fatal(err) - // } - // e.SetLabel("edgeE") - // var buf bytes.Buffer - // if err := g.Render(graph, "dot", &buf); err != nil { - // log.Fatal(err) - // } - - // // print - // fmt.Println(buf.String()) - c.JSON(http.StatusOK, dag) } - -type DAG struct { - Nodes map[string]*Node `json:"nodes"` - Links [][]string `json:"links"` -} - -type Node struct { - Label string `json:"label"` - Stage *pipeline.Stage `json:"stage"` - Steps []*library.Step `json:"steps"` - - // d3 stuff - ID string `json:"id"` - ParentIDs []string `json:"parent_ids"` -} - -type Edge struct { - EdgeID int `json:"edge_id"` - NodeID int `json:"node_id"` -} diff --git a/router/build.go b/router/build.go index 576921e80..22fa7f18c 100644 --- a/router/build.go +++ b/router/build.go @@ -58,7 +58,7 @@ func BuildHandlers(base *gin.RouterGroup) { build.DELETE("", perm.MustPlatformAdmin(), api.DeleteBuild) build.DELETE("/cancel", executors.Establish(), perm.MustWrite(), api.CancelBuild) build.GET("/logs", perm.MustRead(), api.GetBuildLogs) - build.GET("/dag", perm.MustRead(), api.GetBuildDAG) + build.GET("/graph", perm.MustRead(), api.GetBuildGraph) // Service endpoints // * Log endpoints From fecedc6f47b44ccf7ee758102790730c48090bb0 Mon Sep 17 00:00:00 2001 From: davidvader Date: Sun, 16 Jan 2022 17:29:47 -0600 Subject: [PATCH 04/36] feat: stages/steps nested subgraphs --- api/graph.go | 159 +++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 123 insertions(+), 36 deletions(-) diff --git a/api/graph.go b/api/graph.go index 051791e43..39d6d784b 100644 --- a/api/graph.go +++ b/api/graph.go @@ -7,6 +7,7 @@ package api import ( "fmt" "net/http" + "sort" "strings" "github.com/go-vela/server/compiler" @@ -28,24 +29,48 @@ import ( ) // graph contains nodes, and relationships between nodes, or edges. -// a node is a pipeline stage and its relevant steps. -// an edge is a relationship between nodes, defined by the 'needs' tag. +// a graphs is a collection of subgraphs and edges. +// a subgraph is a cluster of nodes. +// a node is a pipeline step. +// an edge is a connection between two nodes on the graph. +// a connection between two nodes is defined by the 'needs' tag. type graph struct { - Nodes map[int]*node `json:"nodes"` - Edges []*edge `json:"edges"` + Subgraphs map[int]*subgraph `json:"subgraphs"` + StageNodes []*stagenode `json:"stage_nodes"` + StageEdges []*edge `json:"stage_edges"` } -// node represents is a pipeline stage and its relevant steps. -type node struct { - Label string `json:"label"` - Stage *pipeline.Stage `json:"stage"` - Steps []*library.Step `json:"steps"` +// subgraph represents is a pipeline stage and its relevant steps. +type subgraph struct { + ID int `json:"id"` + Name string `json:"name"` + StepNodes []*stepnode `json:"step_nodes"` + StepEdges []*edge `json:"step_edges"` + Stage *pipeline.Stage `json:"stage,omitempty"` +} + +// stagenode represents is a pipeline stage and its relevant steps. +type stagenode struct { ID int `json:"id"` + Name string `json:"name"` + Stage *pipeline.Stage `json:"stage,omitempty"` + Steps []*library.Step `json:"steps,omitempty"` +} + +// stepnode represents is a pipeline step and its relevant info. +type stepnode struct { + ID int `json:"id"` + Name string `json:"name"` + Step *library.Step `json:"step,omitempty"` } +// an edge points between two stagenodes. type edge struct { - Source int `json:"source"` - Destination int `json:"destination"` + SourceID int `json:"source_id"` + SourceName string `json:"source_name"` + + DestinationID int `json:"destination_id"` + DestinationName string `json:"destination_name"` } // swagger:operation GET /api/v1/repos/{org}/{repo}/builds/{build}/graph builds GetBuildGraph @@ -209,54 +234,116 @@ func GetBuildGraph(c *gin.Context) { stages[_step.GetStage()] = append(stages[_step.GetStage()], _step) } - // create nodes from pipeline stages - nodes := make(map[int]*node) + // create subgraphs from pipeline stages + subgraphs := make(map[int]*subgraph) + stageNodes := make([]*stagenode, 0) for _, stage := range p.Stages { - // scrub the environment + steps := stages[stage.Name] + if len(steps) == 0 { + // somehow we have a stage with no steps + break + } + + // sort by step number + sort.Slice(steps, func(i int, j int) bool { + return steps[i].GetNumber() < steps[j].GetNumber() + }) + for _, step := range stage.Steps { + // scrub the container environment step.Environment = nil } - nodeID := len(nodes) - node := node{ - Label: stage.Name, - Stage: stage, - Steps: stages[stage.Name], - ID: nodeID, + + stepNodes := make([]*stepnode, 0) + for _, step := range steps { + // build a stepnode + stepNode := stepnode{ + ID: int(step.GetID()), + Name: step.GetName(), + Step: step, + } + stepNodes = append(stepNodes, &stepNode) + } + + stepPairs := getSequencePairs(steps, [][]*library.Step{}) + + stepEdges := make([]*edge, 0) + for _, stepPair := range stepPairs { + if len(stepPair) == 2 { + // build a stepedge + sourceID, sourceName := int(stepPair[0].GetID()), stepPair[0].GetName() + destinationID, destinationName := int(stepPair[1].GetID()), stepPair[1].GetName() + + stepEdge := edge{ + SourceID: sourceID, + SourceName: sourceName, + DestinationID: destinationID, + DestinationName: destinationName, + } + stepEdges = append(stepEdges, &stepEdge) + } + } + + // subgraph ID is the front step ID + subgraphID := int(steps[0].GetID()) + subgraph := subgraph{ + ID: subgraphID, + Name: stage.Name, + StepNodes: stepNodes, + StepEdges: stepEdges, + Stage: stage, + } + subgraphs[subgraphID] = &subgraph + + stageNode := stagenode{ + ID: subgraphID, + Name: stage.Name, + Steps: steps, } - nodes[nodeID] = &node + stageNodes = append(stageNodes, &stageNode) } // create edges from nodes // an edge is as a relationship between two nodes // that is defined by the 'needs' tag - edges := []*edge{} + stageEdges := []*edge{} // loop over nodes - for _, destinationNode := range nodes { + for _, destinationSubgraph := range subgraphs { // compare all nodes against all nodes - for _, sourceNode := range nodes { + for _, sourceSubgraph := range subgraphs { // dont compare the same node - if destinationNode.ID != sourceNode.ID { + if destinationSubgraph.ID != sourceSubgraph.ID { // check destination node needs - for _, need := range (*destinationNode.Stage).Needs { + for _, need := range (*destinationSubgraph.Stage).Needs { // check if destination needs source stage - if sourceNode.Stage.Name == need { - edge := &edge{ - Source: sourceNode.ID, - Destination: destinationNode.ID, + if sourceSubgraph.Stage.Name == need { + stageEdge := edge{ + SourceID: sourceSubgraph.ID, + SourceName: sourceSubgraph.Name, + DestinationID: destinationSubgraph.ID, + DestinationName: destinationSubgraph.Name, } - edges = append(edges, edge) + stageEdges = append(stageEdges, &stageEdge) } } - } } } // construct the response - dag := graph{ - Nodes: nodes, - Edges: edges, + graph := graph{ + Subgraphs: subgraphs, + StageNodes: stageNodes, + StageEdges: stageEdges, } - c.JSON(http.StatusOK, dag) + c.JSON(http.StatusOK, graph) +} + +func getSequencePairs(slice []*library.Step, pairs [][]*library.Step) [][]*library.Step { + if len(slice) > 1 { + pair := []*library.Step{slice[0], slice[1]} + return getSequencePairs(slice[1:], append(pairs, pair)) + } + return pairs } From 8675885ac602ad4c9ae5f122efa273074669738a Mon Sep 17 00:00:00 2001 From: davidvader Date: Sun, 16 Jan 2022 17:38:40 -0600 Subject: [PATCH 05/36] feat: stages & edges --- api/graph.go | 160 +++++++++++++-------------------------------------- 1 file changed, 40 insertions(+), 120 deletions(-) diff --git a/api/graph.go b/api/graph.go index 39d6d784b..554797021 100644 --- a/api/graph.go +++ b/api/graph.go @@ -7,7 +7,6 @@ package api import ( "fmt" "net/http" - "sort" "strings" "github.com/go-vela/server/compiler" @@ -29,48 +28,24 @@ import ( ) // graph contains nodes, and relationships between nodes, or edges. -// a graphs is a collection of subgraphs and edges. -// a subgraph is a cluster of nodes. -// a node is a pipeline step. -// an edge is a connection between two nodes on the graph. -// a connection between two nodes is defined by the 'needs' tag. +// a node is a pipeline stage and its relevant steps. +// an edge is a relationship between nodes, defined by the 'needs' tag. type graph struct { - Subgraphs map[int]*subgraph `json:"subgraphs"` - StageNodes []*stagenode `json:"stage_nodes"` - StageEdges []*edge `json:"stage_edges"` + Nodes map[int]*node `json:"nodes"` + Edges []*edge `json:"edges"` } -// subgraph represents is a pipeline stage and its relevant steps. -type subgraph struct { - ID int `json:"id"` - Name string `json:"name"` - StepNodes []*stepnode `json:"step_nodes"` - StepEdges []*edge `json:"step_edges"` - Stage *pipeline.Stage `json:"stage,omitempty"` -} - -// stagenode represents is a pipeline stage and its relevant steps. -type stagenode struct { - ID int `json:"id"` +// node represents is a pipeline stage and its relevant steps. +type node struct { Name string `json:"name"` - Stage *pipeline.Stage `json:"stage,omitempty"` - Steps []*library.Step `json:"steps,omitempty"` -} - -// stepnode represents is a pipeline step and its relevant info. -type stepnode struct { - ID int `json:"id"` - Name string `json:"name"` - Step *library.Step `json:"step,omitempty"` + Stage *pipeline.Stage `json:"stage"` + Steps []*library.Step `json:"steps"` + ID int `json:"id"` } -// an edge points between two stagenodes. type edge struct { - SourceID int `json:"source_id"` - SourceName string `json:"source_name"` - - DestinationID int `json:"destination_id"` - DestinationName string `json:"destination_name"` + Source int `json:"source"` + Destination int `json:"destination"` } // swagger:operation GET /api/v1/repos/{org}/{repo}/builds/{build}/graph builds GetBuildGraph @@ -234,116 +209,61 @@ func GetBuildGraph(c *gin.Context) { stages[_step.GetStage()] = append(stages[_step.GetStage()], _step) } - // create subgraphs from pipeline stages - subgraphs := make(map[int]*subgraph) - stageNodes := make([]*stagenode, 0) + // create nodes from pipeline stages + nodes := make(map[int]*node) for _, stage := range p.Stages { - steps := stages[stage.Name] - if len(steps) == 0 { - // somehow we have a stage with no steps - break - } - - // sort by step number - sort.Slice(steps, func(i int, j int) bool { - return steps[i].GetNumber() < steps[j].GetNumber() - }) - + // scrub the environment for _, step := range stage.Steps { - // scrub the container environment step.Environment = nil } - - stepNodes := make([]*stepnode, 0) - for _, step := range steps { - // build a stepnode - stepNode := stepnode{ - ID: int(step.GetID()), - Name: step.GetName(), - Step: step, - } - stepNodes = append(stepNodes, &stepNode) - } - - stepPairs := getSequencePairs(steps, [][]*library.Step{}) - - stepEdges := make([]*edge, 0) - for _, stepPair := range stepPairs { - if len(stepPair) == 2 { - // build a stepedge - sourceID, sourceName := int(stepPair[0].GetID()), stepPair[0].GetName() - destinationID, destinationName := int(stepPair[1].GetID()), stepPair[1].GetName() - - stepEdge := edge{ - SourceID: sourceID, - SourceName: sourceName, - DestinationID: destinationID, - DestinationName: destinationName, - } - stepEdges = append(stepEdges, &stepEdge) - } - } - - // subgraph ID is the front step ID - subgraphID := int(steps[0].GetID()) - subgraph := subgraph{ - ID: subgraphID, - Name: stage.Name, - StepNodes: stepNodes, - StepEdges: stepEdges, - Stage: stage, - } - subgraphs[subgraphID] = &subgraph - - stageNode := stagenode{ - ID: subgraphID, + nodeID := len(nodes) + node := node{ Name: stage.Name, - Steps: steps, + Stage: stage, + Steps: stages[stage.Name], + ID: nodeID, } - stageNodes = append(stageNodes, &stageNode) + nodes[nodeID] = &node } // create edges from nodes // an edge is as a relationship between two nodes // that is defined by the 'needs' tag - stageEdges := []*edge{} + edges := []*edge{} // loop over nodes - for _, destinationSubgraph := range subgraphs { + for _, destinationNode := range nodes { // compare all nodes against all nodes - for _, sourceSubgraph := range subgraphs { + for _, sourceNode := range nodes { // dont compare the same node - if destinationSubgraph.ID != sourceSubgraph.ID { + if destinationNode.ID != sourceNode.ID { // check destination node needs - for _, need := range (*destinationSubgraph.Stage).Needs { + for _, need := range (*destinationNode.Stage).Needs { // check if destination needs source stage - if sourceSubgraph.Stage.Name == need { - stageEdge := edge{ - SourceID: sourceSubgraph.ID, - SourceName: sourceSubgraph.Name, - DestinationID: destinationSubgraph.ID, - DestinationName: destinationSubgraph.Name, + if sourceNode.Stage.Name == need { + edge := &edge{ + Source: sourceNode.ID, + Destination: destinationNode.ID, } - stageEdges = append(stageEdges, &stageEdge) + edges = append(edges, edge) } } + } } } + if len(nodes) > 5000 { + c.JSON(http.StatusInternalServerError, "too many nodes on this graph") + } + if len(edges) > 5000 { + c.JSON(http.StatusInternalServerError, "too many edges on this graph") + } + // construct the response graph := graph{ - Subgraphs: subgraphs, - StageNodes: stageNodes, - StageEdges: stageEdges, + Nodes: nodes, + Edges: edges, } c.JSON(http.StatusOK, graph) } - -func getSequencePairs(slice []*library.Step, pairs [][]*library.Step) [][]*library.Step { - if len(slice) > 1 { - pair := []*library.Step{slice[0], slice[1]} - return getSequencePairs(slice[1:], append(pairs, pair)) - } - return pairs -} From c7847d86025a2ee872e85b6c4b6e11379d5c979b Mon Sep 17 00:00:00 2001 From: davidvader Date: Fri, 11 Feb 2022 09:26:22 -0600 Subject: [PATCH 06/36] wip --- api/graph.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/api/graph.go b/api/graph.go index 554797021..6d4ceaa47 100644 --- a/api/graph.go +++ b/api/graph.go @@ -239,7 +239,7 @@ func GetBuildGraph(c *gin.Context) { // check destination node needs for _, need := range (*destinationNode.Stage).Needs { // check if destination needs source stage - if sourceNode.Stage.Name == need { + if sourceNode.Stage.Name == need && need != "clone" { edge := &edge{ Source: sourceNode.ID, Destination: destinationNode.ID, @@ -247,11 +247,12 @@ func GetBuildGraph(c *gin.Context) { edges = append(edges, edge) } } - } } } + // for loop over edges, and collapse same parent edge + if len(nodes) > 5000 { c.JSON(http.StatusInternalServerError, "too many nodes on this graph") } From 117545d3b24325f94cf71810e9636d18629d3a79 Mon Sep 17 00:00:00 2001 From: davidvader Date: Sat, 15 Jul 2023 01:22:57 -0500 Subject: [PATCH 07/36] wip: added built-in nodes and negative ids --- api/build/graph.go | 98 ++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 81 insertions(+), 17 deletions(-) diff --git a/api/build/graph.go b/api/build/graph.go index b9f6d7961..59cf01baf 100644 --- a/api/build/graph.go +++ b/api/build/graph.go @@ -38,15 +38,17 @@ type graph struct { // node represents is a pipeline stage and its relevant steps. type node struct { - Name string `json:"name"` - Stage *pipeline.Stage `json:"stage"` - Steps []*library.Step `json:"steps"` - ID int `json:"id"` + Name string `json:"name"` + Stage *pipeline.Stage `json:"stage"` + Steps []*library.Step `json:"steps"` + ID int `json:"id"` + Status string `json:"status"` } type edge struct { - Source int `json:"source"` - Destination int `json:"destination"` + Source int `json:"source"` + Destination int `json:"destination"` + Status string `json:"status"` } // swagger:operation GET /api/v1/repos/{org}/{repo}/builds/{build}/graph builds GetBuildGraph @@ -106,7 +108,7 @@ func GetBuildGraph(c *gin.Context) { "org": o, "repo": r.GetName(), "user": u.GetName(), - }).Infof("getting all steps for build %s", entry) + }).Infof("getting constructing graph for build %s", entry) // retrieve the steps for the build from the step table steps := []*library.Step{} @@ -171,6 +173,8 @@ func GetBuildGraph(c *gin.Context) { } } + // todo: get pipeline from db instead? + logrus.Info("compiling pipeline") // parse and compile the pipeline configuration file p, _, err := compiler.FromContext(c). @@ -201,30 +205,76 @@ func GetBuildGraph(c *gin.Context) { return } - logrus.Info("creating dag using 'needs'") + logrus.Info("generating build graph") + + type stg struct { + steps []*library.Step + // used for tracking stage status + success int + running int + failure int + } // group library steps by stage name - stages := map[string][]*library.Step{} + stages := map[string]*stg{} for _, _step := range steps { if _, ok := stages[_step.GetStage()]; !ok { - stages[_step.GetStage()] = []*library.Step{} + stages[_step.GetStage()] = &stg{ + steps: []*library.Step{}, + success: 0, + running: 0, + failure: 0, + } + } + switch _step.GetStatus() { + case constants.StatusRunning: + stages[_step.GetStage()].running++ + case constants.StatusSuccess: + stages[_step.GetStage()].success++ + case constants.StatusFailure: + // check if ruleset has 'continue' ? + stages[_step.GetStage()].failure++ + default: } - stages[_step.GetStage()] = append(stages[_step.GetStage()], _step) + stages[_step.GetStage()].steps = append(stages[_step.GetStage()].steps, _step) } // create nodes from pipeline stages nodes := make(map[int]*node) for _, stage := range p.Stages { - // scrub the environment for _, step := range stage.Steps { + // scrub the environment step.Environment = nil } + + // determine the "status" for a stage based on the steps within it. + // this could potentially get complicated with ruleset logic (continue/detach) + status := constants.StatusPending + if stages[stage.Name].running > 0 { + status = constants.StatusRunning + } else if stages[stage.Name].failure > 0 { + status = constants.StatusFailure + } else if stages[stage.Name].success > 0 { + status = constants.StatusSuccess + } + nodeID := len(nodes) + + // override the id for built-in nodes + // this allows for better layout control + if stage.Name == "init" { + nodeID = -3 + } + if stage.Name == "clone" { + nodeID = -2 + } + node := node{ - Name: stage.Name, - Stage: stage, - Steps: stages[stage.Name], - ID: nodeID, + Name: stage.Name, + Stage: stage, + Steps: stages[stage.Name].steps, + ID: nodeID, + Status: status, } nodes[nodeID] = &node } @@ -237,6 +287,20 @@ func GetBuildGraph(c *gin.Context) { for _, destinationNode := range nodes { // compare all nodes against all nodes for _, sourceNode := range nodes { + if sourceNode.ID < 0 && destinationNode.ID < 0 && sourceNode.ID < destinationNode.ID && destinationNode.ID-sourceNode.ID == 1 { + edge := &edge{ + Source: sourceNode.ID, + Destination: destinationNode.ID, + Status: sourceNode.Status, + } + edges = append(edges, edge) + } + + // skip normal processing for built-in nodes + if destinationNode.ID < 0 || sourceNode.ID < 0 { + continue + } + // dont compare the same node if destinationNode.ID != sourceNode.ID { // check destination node needs @@ -246,6 +310,7 @@ func GetBuildGraph(c *gin.Context) { edge := &edge{ Source: sourceNode.ID, Destination: destinationNode.ID, + Status: sourceNode.Status, } edges = append(edges, edge) } @@ -255,7 +320,6 @@ func GetBuildGraph(c *gin.Context) { } // for loop over edges, and collapse same parent edge - if len(nodes) > 5000 { c.JSON(http.StatusInternalServerError, "too many nodes on this graph") } From 82118879587fdb7239f7d72177812bc6adb9e7ae Mon Sep 17 00:00:00 2001 From: davidvader Date: Thu, 3 Aug 2023 08:53:54 -0500 Subject: [PATCH 08/36] wip: add build_id to dag struct --- api/build/graph.go | 97 ++++++++++++++++++++++++++++++++++++---------- 1 file changed, 77 insertions(+), 20 deletions(-) diff --git a/api/build/graph.go b/api/build/graph.go index 59cf01baf..7f9b8df2d 100644 --- a/api/build/graph.go +++ b/api/build/graph.go @@ -32,8 +32,9 @@ import ( // a node is a pipeline stage and its relevant steps. // an edge is a relationship between nodes, defined by the 'needs' tag. type graph struct { - Nodes map[int]*node `json:"nodes"` - Edges []*edge `json:"edges"` + BuildID int64 `json:"build_id"` + Nodes map[int]*node `json:"nodes"` + Edges []*edge `json:"edges"` } // node represents is a pipeline stage and its relevant steps. @@ -214,12 +215,15 @@ func GetBuildGraph(c *gin.Context) { running int failure int } - // group library steps by stage name stages := map[string]*stg{} for _, _step := range steps { - if _, ok := stages[_step.GetStage()]; !ok { - stages[_step.GetStage()] = &stg{ + name := _step.GetStage() + if len(name) == 0 { + name = _step.GetName() + } + if _, ok := stages[name]; !ok { + stages[name] = &stg{ steps: []*library.Step{}, success: 0, running: 0, @@ -228,15 +232,15 @@ func GetBuildGraph(c *gin.Context) { } switch _step.GetStatus() { case constants.StatusRunning: - stages[_step.GetStage()].running++ + stages[name].running++ case constants.StatusSuccess: - stages[_step.GetStage()].success++ + stages[name].success++ case constants.StatusFailure: // check if ruleset has 'continue' ? - stages[_step.GetStage()].failure++ + stages[name].failure++ default: } - stages[_step.GetStage()].steps = append(stages[_step.GetStage()].steps, _step) + stages[name].steps = append(stages[name].steps, _step) } // create nodes from pipeline stages @@ -279,6 +283,49 @@ func GetBuildGraph(c *gin.Context) { nodes[nodeID] = &node } + // no stages so use steps + if len(p.Stages) == 0 { + for _, step := range p.Steps { + // scrub the environment + step.Environment = nil + + stage := &pipeline.Stage{ + Name: step.Name, + } + + // determine the "status" for a stage based on the steps within it. + // this could potentially get complicated with ruleset logic (continue/detach) + status := constants.StatusPending + if stages[stage.Name].running > 0 { + status = constants.StatusRunning + } else if stages[stage.Name].failure > 0 { + status = constants.StatusFailure + } else if stages[stage.Name].success > 0 { + status = constants.StatusSuccess + } + + nodeID := len(nodes) + + // override the id for built-in nodes + // this allows for better layout control + // if stage.Name == "init" { + // nodeID = -3 + // } + // if stage.Name == "clone" { + // nodeID = -2 + // } + + node := node{ + Name: stage.Name, + Stage: stage, + Steps: stages[stage.Name].steps, + ID: nodeID, + Status: status, + } + nodes[nodeID] = &node + } + } + // create edges from nodes // an edge is as a relationship between two nodes // that is defined by the 'needs' tag @@ -303,17 +350,26 @@ func GetBuildGraph(c *gin.Context) { // dont compare the same node if destinationNode.ID != sourceNode.ID { - // check destination node needs - for _, need := range (*destinationNode.Stage).Needs { - // check if destination needs source stage - if sourceNode.Stage.Name == need && need != "clone" { - edge := &edge{ - Source: sourceNode.ID, - Destination: destinationNode.ID, - Status: sourceNode.Status, + if len((*destinationNode.Stage).Needs) > 0 { + // check destination node needs + for _, need := range (*destinationNode.Stage).Needs { + // check if destination needs source stage + if sourceNode.Stage.Name == need && need != "clone" { + edge := &edge{ + Source: sourceNode.ID, + Destination: destinationNode.ID, + Status: sourceNode.Status, + } + edges = append(edges, edge) } - edges = append(edges, edge) } + } else { + edge := &edge{ + Source: sourceNode.ID, + Destination: sourceNode.ID + 1, + Status: sourceNode.Status, + } + edges = append(edges, edge) } } } @@ -329,8 +385,9 @@ func GetBuildGraph(c *gin.Context) { // construct the response graph := graph{ - Nodes: nodes, - Edges: edges, + BuildID: b.GetID(), + Nodes: nodes, + Edges: edges, } c.JSON(http.StatusOK, graph) From 0dc60008204eaff5c35d7f8390c294b54c3016c9 Mon Sep 17 00:00:00 2001 From: davidvader Date: Fri, 29 Sep 2023 09:37:53 -0500 Subject: [PATCH 09/36] chore: merge with main --- .gitignore | 1 + api/build/graph.go | 5 +++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index c557bc18b..29b233934 100644 --- a/.gitignore +++ b/.gitignore @@ -48,6 +48,7 @@ api-spec.json # Edit at https://www.toptal.com/developers/gitignore?templates=visualstudiocode ### VisualStudioCode ### +.vscode/ .vscode/* !.vscode/settings.json !.vscode/tasks.json diff --git a/api/build/graph.go b/api/build/graph.go index 7f9b8df2d..db1a8428e 100644 --- a/api/build/graph.go +++ b/api/build/graph.go @@ -98,6 +98,7 @@ func GetBuildGraph(c *gin.Context) { r := repo.Retrieve(c) u := user.Retrieve(c) m := c.MustGet("metadata").(*types.Metadata) + ctx := c.Request.Context() entry := fmt.Sprintf("%s/%d", r.GetFullName(), b.GetNumber()) @@ -148,7 +149,7 @@ func GetBuildGraph(c *gin.Context) { baseErr := "unable to generate build graph" // send API call to capture the pipeline configuration file - config, err := scm.FromContext(c).ConfigBackoff(u, r, b.GetCommit()) + config, err := scm.FromContext(c).ConfigBackoff(ctx, u, r, b.GetCommit()) if err != nil { // nolint: lll // ignore long line length due to error message retErr := fmt.Errorf("%s: failed to get pipeline configuration for %s: %v", baseErr, r.GetFullName(), err) @@ -164,7 +165,7 @@ func GetBuildGraph(c *gin.Context) { // check if the build event is not pull_request if !strings.EqualFold(b.GetEvent(), constants.EventPull) { // send API call to capture list of files changed for the commit - files, err = scm.FromContext(c).Changeset(u, r, b.GetCommit()) + files, err = scm.FromContext(c).Changeset(ctx, u, r, b.GetCommit()) if err != nil { retErr := fmt.Errorf("%s: failed to get changeset for %s: %v", baseErr, r.GetFullName(), err) util.HandleError(c, http.StatusInternalServerError, retErr) From e80b5cdf6fa21d322119fc801d4e20072f23fb0d Mon Sep 17 00:00:00 2001 From: davidvader Date: Fri, 6 Oct 2023 09:10:15 -0500 Subject: [PATCH 10/36] feat: services --- api/build/graph.go | 83 ++++++++++++++++++++++++++++++++++++++++------ docker-compose.yml | 2 +- 2 files changed, 74 insertions(+), 11 deletions(-) diff --git a/api/build/graph.go b/api/build/graph.go index db1a8428e..afc12e05b 100644 --- a/api/build/graph.go +++ b/api/build/graph.go @@ -37,19 +37,22 @@ type graph struct { Edges []*edge `json:"edges"` } -// node represents is a pipeline stage and its relevant steps. +// node represents a pipeline stage and its relevant steps. type node struct { - Name string `json:"name"` - Stage *pipeline.Stage `json:"stage"` - Steps []*library.Step `json:"steps"` - ID int `json:"id"` - Status string `json:"status"` + Name string `json:"name"` + Stage *pipeline.Stage `json:"stage"` + Steps []*library.Step `json:"steps"` + ID int `json:"id"` + Status string `json:"status"` + Service *library.Service `json:"service"` } +// an edge is a relationship between nodes, defined by the 'needs' tag. type edge struct { Source int `json:"source"` Destination int `json:"destination"` Status string `json:"status"` + Service bool `json:"service"` } // swagger:operation GET /api/v1/repos/{org}/{repo}/builds/{build}/graph builds GetBuildGraph @@ -144,11 +147,38 @@ func GetBuildGraph(c *gin.Context) { return } + // retrieve the services for the build from the service table + services := []*library.Service{} + page = 1 + perPage = 100 + for page > 0 { + // retrieve build services (per page) from the database + servicesPart, _, err := database.FromContext(c).ListServicesForBuild(ctx, b, nil, page, perPage) + if err != nil { + retErr := fmt.Errorf("unable to retrieve services for build %s: %w", entry, err) + util.HandleError(c, http.StatusNotFound, retErr) + return + } + + // add page of services to list services + services = append(services, servicesPart...) + + // assume no more pages exist if under 100 results are returned + // + // nolint: gomnd // ignore magic number + if len(servicesPart) < 100 { + page = 0 + } else { + page++ + } + } + logrus.Info("retrieving pipeline configuration file") baseErr := "unable to generate build graph" // send API call to capture the pipeline configuration file + // todo: pipeline table config, err := scm.FromContext(c).ConfigBackoff(ctx, u, r, b.GetCommit()) if err != nil { // nolint: lll // ignore long line length due to error message @@ -246,6 +276,35 @@ func GetBuildGraph(c *gin.Context) { // create nodes from pipeline stages nodes := make(map[int]*node) + + // create edges from nodes + // an edge is as a relationship between two nodes + // that is defined by the 'needs' tag + edges := []*edge{} + + for _, service := range services { + // scrub the environment + nodeID := len(nodes) + node := node{ + Name: service.GetName(), + ID: nodeID, + Status: service.GetStatus(), + Service: service, + } + nodes[nodeID] = &node + + // group services using invisible edges + if nodeID > 0 { + edge := &edge{ + Source: nodeID - 1, + Destination: nodeID, + Status: service.GetStatus(), + Service: true, + } + edges = append(edges, edge) + } + } + for _, stage := range p.Stages { for _, step := range stage.Steps { // scrub the environment @@ -327,14 +386,16 @@ func GetBuildGraph(c *gin.Context) { } } - // create edges from nodes - // an edge is as a relationship between two nodes - // that is defined by the 'needs' tag - edges := []*edge{} // loop over nodes for _, destinationNode := range nodes { + if destinationNode.Stage == nil { + continue + } // compare all nodes against all nodes for _, sourceNode := range nodes { + if sourceNode.Stage == nil { + continue + } if sourceNode.ID < 0 && destinationNode.ID < 0 && sourceNode.ID < destinationNode.ID && destinationNode.ID-sourceNode.ID == 1 { edge := &edge{ Source: sourceNode.ID, @@ -377,6 +438,7 @@ func GetBuildGraph(c *gin.Context) { } // for loop over edges, and collapse same parent edge + // todo: move this check above the processing ? if len(nodes) > 5000 { c.JSON(http.StatusInternalServerError, "too many nodes on this graph") } @@ -391,5 +453,6 @@ func GetBuildGraph(c *gin.Context) { Edges: edges, } + // todo: cli to generate digraph? output in format that can be used in other visualizers? c.JSON(http.StatusOK, graph) } diff --git a/docker-compose.yml b/docker-compose.yml index 02b749ed4..8a1d1ae80 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -29,6 +29,7 @@ services: QUEUE_DRIVER: redis QUEUE_ADDR: 'redis://redis:6379' QUEUE_PRIVATE_KEY: 'tCIevHOBq6DdN5SSBtteXUusjjd0fOqzk2eyi0DMq04NewmShNKQeUbbp3vkvIckb4pCxc+vxUo+mYf/vzOaSg==' + QUEUE_PUBLIC_KEY: 'DXsJkoTSkHlG26d75LyHJG+KQsXPr8VKPpmH/78zmko=' SCM_DRIVER: github SCM_CONTEXT: 'continuous-integration/vela' SECRET_VAULT: 'true' @@ -39,7 +40,6 @@ services: VELA_LOG_LEVEL: trace # comment the line below to use registration flow VELA_SECRET: 'zB7mrKDTZqNeNTD8z47yG4DHywspAh' - QUEUE_PUBLIC_KEY: 'DXsJkoTSkHlG26d75LyHJG+KQsXPr8VKPpmH/78zmko=' VELA_SERVER_PRIVATE_KEY: 'F534FF2A080E45F38E05DC70752E6787' VELA_USER_REFRESH_TOKEN_DURATION: 90m VELA_USER_ACCESS_TOKEN_DURATION: 60m From eb321eb68f0c63cdc9545e7782c6564514f21ef4 Mon Sep 17 00:00:00 2001 From: davidvader Date: Tue, 10 Oct 2023 16:52:19 -0500 Subject: [PATCH 11/36] wip: established required code, clean-up phase --- api/build/graph.go | 281 ++++++++++++++++++++++++++++----------------- 1 file changed, 176 insertions(+), 105 deletions(-) diff --git a/api/build/graph.go b/api/build/graph.go index afc12e05b..d2ac027f6 100644 --- a/api/build/graph.go +++ b/api/build/graph.go @@ -39,22 +39,48 @@ type graph struct { // node represents a pipeline stage and its relevant steps. type node struct { - Name string `json:"name"` - Stage *pipeline.Stage `json:"stage"` - Steps []*library.Step `json:"steps"` - ID int `json:"id"` - Status string `json:"status"` - Service *library.Service `json:"service"` + Cluster int `json:"cluster"` + ID int `json:"id"` + Name string `json:"name"` + + // vela metadata + Status string `json:"status"` + Duration int `json:"duration"` + Steps []*library.Step `json:"steps"` + + // unexported data used for building edges + Stage *pipeline.Stage `json:"-"` } -// an edge is a relationship between nodes, defined by the 'needs' tag. +// edge represents a relationship between nodes, primarily defined by a stage 'needs' tag. type edge struct { - Source int `json:"source"` - Destination int `json:"destination"` - Status string `json:"status"` - Service bool `json:"service"` + Cluster int `json:"cluster"` + Source int `json:"source"` + Destination int `json:"destination"` + + // vela metadata + Status string `json:"status"` +} + +// stg represents a stage's steps and some metadata for producing node/edge information +type stg struct { + steps []*library.Step + // used for tracking stage status + success int + running int + failure int + killed int + startedAt int + finishedAt int } +const ( + // clusters determine graph orientation + BuiltInCluster = 2 + PipelineCluster = 1 + ServiceCluster = 0 +) + // swagger:operation GET /api/v1/repos/{org}/{repo}/builds/{build}/graph builds GetBuildGraph // // Get directed a-cyclical graph for a build in the configured backend @@ -100,7 +126,9 @@ func GetBuildGraph(c *gin.Context) { o := org.Retrieve(c) r := repo.Retrieve(c) u := user.Retrieve(c) + m := c.MustGet("metadata").(*types.Metadata) + ctx := c.Request.Context() entry := fmt.Sprintf("%s/%d", r.GetFullName(), b.GetNumber()) @@ -173,16 +201,29 @@ func GetBuildGraph(c *gin.Context) { } } - logrus.Info("retrieving pipeline configuration file") - baseErr := "unable to generate build graph" - // send API call to capture the pipeline configuration file - // todo: pipeline table - config, err := scm.FromContext(c).ConfigBackoff(ctx, u, r, b.GetCommit()) - if err != nil { - // nolint: lll // ignore long line length due to error message - retErr := fmt.Errorf("%s: failed to get pipeline configuration for %s: %v", baseErr, r.GetFullName(), err) + logrus.Info("retrieving pipeline configuration") + var config []byte + + lp, err := database.FromContext(c).GetPipelineForRepo(ctx, b.GetCommit(), r) + if err != nil { // assume the pipeline doesn't exist in the database yet (before pipeline support was added) + // send API call to capture the pipeline configuration file + config, err = scm.FromContext(c).ConfigBackoff(ctx, u, r, b.GetCommit()) + if err != nil { + retErr := fmt.Errorf("unable to get pipeline configuration for %s: %w", r.GetFullName(), err) + + util.HandleError(c, http.StatusNotFound, retErr) + + return + } + } else { + config = lp.GetData() + } + + if config == nil { + retErr := fmt.Errorf("unable to get pipeline configuration for %s: config is nil", r.GetFullName()) + util.HandleError(c, http.StatusNotFound, retErr) return @@ -205,9 +246,8 @@ func GetBuildGraph(c *gin.Context) { } } - // todo: get pipeline from db instead? + logrus.Info("compiling pipeline configuration") - logrus.Info("compiling pipeline") // parse and compile the pipeline configuration file p, _, err := compiler.FromContext(c). Duplicate(). @@ -239,40 +279,7 @@ func GetBuildGraph(c *gin.Context) { logrus.Info("generating build graph") - type stg struct { - steps []*library.Step - // used for tracking stage status - success int - running int - failure int - } - // group library steps by stage name - stages := map[string]*stg{} - for _, _step := range steps { - name := _step.GetStage() - if len(name) == 0 { - name = _step.GetName() - } - if _, ok := stages[name]; !ok { - stages[name] = &stg{ - steps: []*library.Step{}, - success: 0, - running: 0, - failure: 0, - } - } - switch _step.GetStatus() { - case constants.StatusRunning: - stages[name].running++ - case constants.StatusSuccess: - stages[name].success++ - case constants.StatusFailure: - // check if ruleset has 'continue' ? - stages[name].failure++ - default: - } - stages[name].steps = append(stages[name].steps, _step) - } + stages := p.Stages // create nodes from pipeline stages nodes := make(map[int]*node) @@ -282,64 +289,94 @@ func GetBuildGraph(c *gin.Context) { // that is defined by the 'needs' tag edges := []*edge{} + // initialize a map for grouping steps by stage name + // and tracking stage information + stageMap := map[string]*stg{} + for _, _step := range steps { + name := _step.GetStage() + if len(name) == 0 { + name = _step.GetName() + } + if _, ok := stageMap[name]; !ok { + stageMap[name] = &stg{ + steps: []*library.Step{}, + success: 0, + running: 0, + failure: 0, + killed: 0, + startedAt: 0, + finishedAt: 0, + } + } + stageMap[name].updateStgTracker(_step) + stageMap[name].steps = append(stageMap[name].steps, _step) + } + for _, service := range services { - // scrub the environment nodeID := len(nodes) node := node{ - Name: service.GetName(), + // set service cluster + Cluster: ServiceCluster, ID: nodeID, - Status: service.GetStatus(), - Service: service, + + Name: service.GetName(), + Status: service.GetStatus(), + // todo: runtime + Duration: int(service.GetFinished() - service.GetStarted()), + // todo: is this right? + // Steps: []string{}, } nodes[nodeID] = &node // group services using invisible edges if nodeID > 0 { edge := &edge{ + // set service cluster + Cluster: ServiceCluster, + // link them together for visual effect Source: nodeID - 1, Destination: nodeID, - Status: service.GetStatus(), - Service: true, + + Status: service.GetStatus(), } edges = append(edges, edge) } } - for _, stage := range p.Stages { + for _, stage := range stages { for _, step := range stage.Steps { // scrub the environment step.Environment = nil } + nodeID := len(nodes) + // determine the "status" for a stage based on the steps within it. // this could potentially get complicated with ruleset logic (continue/detach) - status := constants.StatusPending - if stages[stage.Name].running > 0 { - status = constants.StatusRunning - } else if stages[stage.Name].failure > 0 { - status = constants.StatusFailure - } else if stages[stage.Name].success > 0 { - status = constants.StatusSuccess - } + status := stageStatus(stageMap[stage.Name]) - nodeID := len(nodes) + node := node{ + Cluster: PipelineCluster, + ID: nodeID, + + Name: stage.Name, + Status: status, + Duration: int(stageMap[stage.Name].finishedAt - stageMap[stage.Name].startedAt), + Steps: stageMap[stage.Name].steps, + + // used for edge creation + Stage: stage, + } // override the id for built-in nodes // this allows for better layout control if stage.Name == "init" { - nodeID = -3 + node.Cluster = BuiltInCluster } if stage.Name == "clone" { - nodeID = -2 + node.Cluster = BuiltInCluster } - node := node{ - Name: stage.Name, - Stage: stage, - Steps: stages[stage.Name].steps, - ID: nodeID, - Status: status, - } nodes[nodeID] = &node } @@ -349,55 +386,50 @@ func GetBuildGraph(c *gin.Context) { // scrub the environment step.Environment = nil + // mock stage for edge creation stage := &pipeline.Stage{ Name: step.Name, } // determine the "status" for a stage based on the steps within it. // this could potentially get complicated with ruleset logic (continue/detach) - status := constants.StatusPending - if stages[stage.Name].running > 0 { - status = constants.StatusRunning - } else if stages[stage.Name].failure > 0 { - status = constants.StatusFailure - } else if stages[stage.Name].success > 0 { - status = constants.StatusSuccess - } + status := stageStatus(stageMap[stage.Name]) nodeID := len(nodes) - // override the id for built-in nodes - // this allows for better layout control - // if stage.Name == "init" { - // nodeID = -3 - // } - // if stage.Name == "clone" { - // nodeID = -2 - // } - node := node{ - Name: stage.Name, - Stage: stage, - Steps: stages[stage.Name].steps, - ID: nodeID, - Status: status, + Cluster: PipelineCluster, + ID: nodeID, + + Name: stage.Name, + Status: status, + Duration: int(stageMap[stage.Name].finishedAt - stageMap[stage.Name].startedAt), + Steps: stageMap[stage.Name].steps, + + // used for edge creation + Stage: stage, } nodes[nodeID] = &node } } + // done building nodes - // loop over nodes + // loop over nodes and create edges based on 'needs' for _, destinationNode := range nodes { + // if theres no stage, skip because the edge is already created? if destinationNode.Stage == nil { continue } + // compare all nodes against all nodes for _, sourceNode := range nodes { if sourceNode.Stage == nil { continue } - if sourceNode.ID < 0 && destinationNode.ID < 0 && sourceNode.ID < destinationNode.ID && destinationNode.ID-sourceNode.ID == 1 { + + if sourceNode.Cluster == BuiltInCluster && destinationNode.Cluster == BuiltInCluster && sourceNode.ID < destinationNode.ID && destinationNode.ID-sourceNode.ID == 1 { edge := &edge{ + Cluster: sourceNode.Cluster, Source: sourceNode.ID, Destination: destinationNode.ID, Status: sourceNode.Status, @@ -406,7 +438,7 @@ func GetBuildGraph(c *gin.Context) { } // skip normal processing for built-in nodes - if destinationNode.ID < 0 || sourceNode.ID < 0 { + if destinationNode.Cluster == BuiltInCluster || sourceNode.Cluster == BuiltInCluster { continue } @@ -418,6 +450,7 @@ func GetBuildGraph(c *gin.Context) { // check if destination needs source stage if sourceNode.Stage.Name == need && need != "clone" { edge := &edge{ + Cluster: sourceNode.Cluster, Source: sourceNode.ID, Destination: destinationNode.ID, Status: sourceNode.Status, @@ -427,6 +460,7 @@ func GetBuildGraph(c *gin.Context) { } } else { edge := &edge{ + Cluster: sourceNode.Cluster, Source: sourceNode.ID, Destination: sourceNode.ID + 1, Status: sourceNode.Status, @@ -442,6 +476,7 @@ func GetBuildGraph(c *gin.Context) { if len(nodes) > 5000 { c.JSON(http.StatusInternalServerError, "too many nodes on this graph") } + if len(edges) > 5000 { c.JSON(http.StatusInternalServerError, "too many edges on this graph") } @@ -456,3 +491,39 @@ func GetBuildGraph(c *gin.Context) { // todo: cli to generate digraph? output in format that can be used in other visualizers? c.JSON(http.StatusOK, graph) } + +func (s *stg) updateStgTracker(step *library.Step) { + switch step.GetStatus() { + case constants.StatusRunning: + s.running++ + case constants.StatusSuccess: + s.success++ + case constants.StatusFailure: + // check if ruleset has 'continue' ? + s.failure++ + case constants.StatusKilled: + s.killed++ + default: + } + + if s.finishedAt == 0 || s.finishedAt < int(step.GetFinished()) { + s.finishedAt = int(step.GetFinished()) + } + if s.startedAt == 0 || s.startedAt > int(step.GetStarted()) { + s.startedAt = int(step.GetStarted()) + } +} + +func stageStatus(s *stg) string { + status := constants.StatusPending + if s.running > 0 { + status = constants.StatusRunning + } else if s.failure > 0 { + status = constants.StatusFailure + } else if s.success > 0 { + status = constants.StatusSuccess + } else if s.killed > 0 { + status = constants.StatusKilled + } + return status +} From de697d65b144061e2a3d5369b57bae31a9f3a24c Mon Sep 17 00:00:00 2001 From: davidvader Date: Wed, 11 Oct 2023 10:28:51 -0500 Subject: [PATCH 12/36] tweak: return on complexity error --- api/build/graph.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/api/build/graph.go b/api/build/graph.go index d2ac027f6..635fb183d 100644 --- a/api/build/graph.go +++ b/api/build/graph.go @@ -368,7 +368,7 @@ func GetBuildGraph(c *gin.Context) { Stage: stage, } - // override the id for built-in nodes + // override the cluster id for built-in nodes // this allows for better layout control if stage.Name == "init" { node.Cluster = BuiltInCluster @@ -475,10 +475,12 @@ func GetBuildGraph(c *gin.Context) { // todo: move this check above the processing ? if len(nodes) > 5000 { c.JSON(http.StatusInternalServerError, "too many nodes on this graph") + return } if len(edges) > 5000 { c.JSON(http.StatusInternalServerError, "too many edges on this graph") + return } // construct the response From da97cae929cc3b9719c376684d2ea97cc2cd5211 Mon Sep 17 00:00:00 2001 From: davidvader Date: Thu, 12 Oct 2023 18:20:23 -0500 Subject: [PATCH 13/36] chore: clean up code --- api/build/graph.go | 293 +++++++++++++++++++++++++-------------------- 1 file changed, 161 insertions(+), 132 deletions(-) diff --git a/api/build/graph.go b/api/build/graph.go index 635fb183d..b7a724c4a 100644 --- a/api/build/graph.go +++ b/api/build/graph.go @@ -39,14 +39,15 @@ type graph struct { // node represents a pipeline stage and its relevant steps. type node struct { - Cluster int `json:"cluster"` ID int `json:"id"` + Cluster int `json:"cluster"` Name string `json:"name"` // vela metadata - Status string `json:"status"` - Duration int `json:"duration"` - Steps []*library.Step `json:"steps"` + Status string `json:"status"` + StartedAt int `json:"started_at"` + FinishedAt int `json:"finished_at"` + Steps []*library.Step `json:"steps"` // unexported data used for building edges Stage *pipeline.Stage `json:"-"` @@ -126,22 +127,23 @@ func GetBuildGraph(c *gin.Context) { o := org.Retrieve(c) r := repo.Retrieve(c) u := user.Retrieve(c) - m := c.MustGet("metadata").(*types.Metadata) - ctx := c.Request.Context() entry := fmt.Sprintf("%s/%d", r.GetFullName(), b.GetNumber()) - - // update engine logger with API metadata - // - // https://pkg.go.dev/github.com/sirupsen/logrus?tab=doc#Entry.WithFields - logrus.WithFields(logrus.Fields{ + logger := logrus.WithFields(logrus.Fields{ "build": b.GetNumber(), "org": o, "repo": r.GetName(), "user": u.GetName(), - }).Infof("getting constructing graph for build %s", entry) + }) + + baseErr := "unable to construct graph" + + // update engine logger with API metadata + // + // https://pkg.go.dev/github.com/sirupsen/logrus?tab=doc#Entry.WithFields + logger.Infof("constructing graph for build %s", entry) // retrieve the steps for the build from the step table steps := []*library.Step{} @@ -152,7 +154,9 @@ func GetBuildGraph(c *gin.Context) { stepsPart, _, err := database.FromContext(c).ListStepsForBuild(b, nil, page, perPage) if err != nil { retErr := fmt.Errorf("unable to retrieve steps for build %s: %w", entry, err) + util.HandleError(c, http.StatusNotFound, retErr) + return } @@ -184,7 +188,9 @@ func GetBuildGraph(c *gin.Context) { servicesPart, _, err := database.FromContext(c).ListServicesForBuild(ctx, b, nil, page, perPage) if err != nil { retErr := fmt.Errorf("unable to retrieve services for build %s: %w", entry, err) + util.HandleError(c, http.StatusNotFound, retErr) + return } @@ -201,9 +207,8 @@ func GetBuildGraph(c *gin.Context) { } } - baseErr := "unable to generate build graph" + logger.Info("retrieving pipeline configuration") - logrus.Info("retrieving pipeline configuration") var config []byte lp, err := database.FromContext(c).GetPipelineForRepo(ctx, b.GetCommit(), r) @@ -239,6 +244,7 @@ func GetBuildGraph(c *gin.Context) { files, err = scm.FromContext(c).Changeset(ctx, u, r, b.GetCommit()) if err != nil { retErr := fmt.Errorf("%s: failed to get changeset for %s: %v", baseErr, r.GetFullName(), err) + util.HandleError(c, http.StatusInternalServerError, retErr) return @@ -246,7 +252,7 @@ func GetBuildGraph(c *gin.Context) { } } - logrus.Info("compiling pipeline configuration") + logger.Info("compiling pipeline configuration") // parse and compile the pipeline configuration file p, _, err := compiler.FromContext(c). @@ -261,10 +267,10 @@ func GetBuildGraph(c *gin.Context) { // format the error message with extra information err = fmt.Errorf("unable to compile pipeline configuration for %s: %v", r.GetFullName(), err) - // log the error for traceability - logrus.Error(err.Error()) + logger.Error(err.Error()) retErr := fmt.Errorf("%s: %v", baseErr, err) + util.HandleError(c, http.StatusInternalServerError, retErr) return @@ -277,9 +283,7 @@ func GetBuildGraph(c *gin.Context) { return } - logrus.Info("generating build graph") - - stages := p.Stages + logger.Info("generating build graph") // create nodes from pipeline stages nodes := make(map[int]*node) @@ -292,11 +296,13 @@ func GetBuildGraph(c *gin.Context) { // initialize a map for grouping steps by stage name // and tracking stage information stageMap := map[string]*stg{} - for _, _step := range steps { - name := _step.GetStage() + for _, step := range steps { + name := step.GetStage() if len(name) == 0 { - name = _step.GetName() + name = step.GetName() } + + // initialize a stage tracker if _, ok := stageMap[name]; !ok { stageMap[name] = &stg{ steps: []*library.Step{}, @@ -308,79 +314,80 @@ func GetBuildGraph(c *gin.Context) { finishedAt: 0, } } - stageMap[name].updateStgTracker(_step) - stageMap[name].steps = append(stageMap[name].steps, _step) + + // retrieve the stage to update + s := stageMap[name] + + // count each step status + switch step.GetStatus() { + case constants.StatusRunning: + s.running++ + case constants.StatusSuccess: + s.success++ + case constants.StatusFailure: + s.failure++ + case constants.StatusKilled: + s.killed++ + default: + } + + // keep track of the widest time window possible + if s.finishedAt == 0 || s.finishedAt < int(step.GetFinished()) { + s.finishedAt = int(step.GetFinished()) + } + if s.startedAt == 0 || s.startedAt > int(step.GetStarted()) { + s.startedAt = int(step.GetStarted()) + } + + s.steps = append(s.steps, step) } + // construct services nodes separately + // services are grouped via cluster and manually-constructed edges for _, service := range services { + // create the node nodeID := len(nodes) - node := node{ - // set service cluster - Cluster: ServiceCluster, - ID: nodeID, - - Name: service.GetName(), - Status: service.GetStatus(), - // todo: runtime - Duration: int(service.GetFinished() - service.GetStarted()), - // todo: is this right? - // Steps: []string{}, - } - nodes[nodeID] = &node + node := nodeFromService(nodeID, service) + nodes[nodeID] = node - // group services using invisible edges + // create a sequential edge for nodes after the first if nodeID > 0 { edge := &edge{ - // set service cluster - Cluster: ServiceCluster, - // link them together for visual effect + Cluster: ServiceCluster, Source: nodeID - 1, Destination: nodeID, - - Status: service.GetStatus(), + Status: service.GetStatus(), } edges = append(edges, edge) } } - for _, stage := range stages { + // construct pipeline stages nodes when stages exist + for _, stage := range p.Stages { + // scrub the environment for _, step := range stage.Steps { - // scrub the environment step.Environment = nil } + // create the node nodeID := len(nodes) - // determine the "status" for a stage based on the steps within it. - // this could potentially get complicated with ruleset logic (continue/detach) - status := stageStatus(stageMap[stage.Name]) - - node := node{ - Cluster: PipelineCluster, - ID: nodeID, - - Name: stage.Name, - Status: status, - Duration: int(stageMap[stage.Name].finishedAt - stageMap[stage.Name].startedAt), - Steps: stageMap[stage.Name].steps, - - // used for edge creation - Stage: stage, - } + cluster := PipelineCluster // override the cluster id for built-in nodes - // this allows for better layout control + // this allows for better layout control because all stages need 'clone' if stage.Name == "init" { - node.Cluster = BuiltInCluster + cluster = BuiltInCluster } if stage.Name == "clone" { - node.Cluster = BuiltInCluster + cluster = BuiltInCluster } - nodes[nodeID] = &node + node := nodeFromStage(nodeID, cluster, stage, stageMap[stage.Name]) + nodes[nodeID] = node } - // no stages so use steps + // create single-step stages when no stages exist if len(p.Stages) == 0 { for _, step := range p.Steps { // scrub the environment @@ -388,33 +395,22 @@ func GetBuildGraph(c *gin.Context) { // mock stage for edge creation stage := &pipeline.Stage{ - Name: step.Name, + Name: step.Name, + Needs: []string{}, } - // determine the "status" for a stage based on the steps within it. - // this could potentially get complicated with ruleset logic (continue/detach) - status := stageStatus(stageMap[stage.Name]) - + // create the node nodeID := len(nodes) - node := node{ - Cluster: PipelineCluster, - ID: nodeID, + // no built-in step separation for graphs without stages + cluster := PipelineCluster - Name: stage.Name, - Status: status, - Duration: int(stageMap[stage.Name].finishedAt - stageMap[stage.Name].startedAt), - Steps: stageMap[stage.Name].steps, - - // used for edge creation - Stage: stage, - } - nodes[nodeID] = &node + node := nodeFromStage(nodeID, cluster, stage, stageMap[stage.Name]) + nodes[nodeID] = node } } - // done building nodes - // loop over nodes and create edges based on 'needs' + // loop over all nodes and create edges based on 'needs' for _, destinationNode := range nodes { // if theres no stage, skip because the edge is already created? if destinationNode.Stage == nil { @@ -427,7 +423,11 @@ func GetBuildGraph(c *gin.Context) { continue } - if sourceNode.Cluster == BuiltInCluster && destinationNode.Cluster == BuiltInCluster && sourceNode.ID < destinationNode.ID && destinationNode.ID-sourceNode.ID == 1 { + // create a sequential edge for built-in nodes + if sourceNode.Cluster == BuiltInCluster && + destinationNode.Cluster == BuiltInCluster && + sourceNode.ID < destinationNode.ID && + destinationNode.ID-sourceNode.ID == 1 { edge := &edge{ Cluster: sourceNode.Cluster, Source: sourceNode.ID, @@ -438,35 +438,51 @@ func GetBuildGraph(c *gin.Context) { } // skip normal processing for built-in nodes - if destinationNode.Cluster == BuiltInCluster || sourceNode.Cluster == BuiltInCluster { + if destinationNode.Cluster == BuiltInCluster { + continue + } + + if sourceNode.Cluster == BuiltInCluster { continue } // dont compare the same node - if destinationNode.ID != sourceNode.ID { - if len((*destinationNode.Stage).Needs) > 0 { - // check destination node needs - for _, need := range (*destinationNode.Stage).Needs { - // check if destination needs source stage - if sourceNode.Stage.Name == need && need != "clone" { - edge := &edge{ - Cluster: sourceNode.Cluster, - Source: sourceNode.ID, - Destination: destinationNode.ID, - Status: sourceNode.Status, - } - edges = append(edges, edge) - } + if destinationNode.ID == sourceNode.ID { + continue + } + + // use needs to create an edge + if len((*destinationNode.Stage).Needs) > 0 { + // check destination node needs + for _, need := range (*destinationNode.Stage).Needs { + // all stages need "clone" + if need == "clone" { + continue } - } else { + + // check destination needs source stage + if sourceNode.Stage.Name != need { + continue + } + + // create an edge for source->destination edge := &edge{ Cluster: sourceNode.Cluster, Source: sourceNode.ID, - Destination: sourceNode.ID + 1, + Destination: destinationNode.ID, Status: sourceNode.Status, } edges = append(edges, edge) } + } else { + // create an edge for source->next + edge := &edge{ + Cluster: sourceNode.Cluster, + Source: sourceNode.ID, + Destination: sourceNode.ID + 1, + Status: sourceNode.Status, + } + edges = append(edges, edge) } } } @@ -490,42 +506,55 @@ func GetBuildGraph(c *gin.Context) { Edges: edges, } - // todo: cli to generate digraph? output in format that can be used in other visualizers? c.JSON(http.StatusOK, graph) } -func (s *stg) updateStgTracker(step *library.Step) { - switch step.GetStatus() { - case constants.StatusRunning: - s.running++ - case constants.StatusSuccess: - s.success++ - case constants.StatusFailure: - // check if ruleset has 'continue' ? - s.failure++ - case constants.StatusKilled: - s.killed++ - default: +// nodeFromStage returns a new node from a stage +func nodeFromStage(nodeID, cluster int, stage *pipeline.Stage, s *stg) *node { + return &node{ + ID: nodeID, + Cluster: cluster, + Name: stage.Name, + Stage: stage, + Steps: s.steps, + Status: s.GetOverallStatus(), + StartedAt: int(s.startedAt), + FinishedAt: int(s.finishedAt), } +} - if s.finishedAt == 0 || s.finishedAt < int(step.GetFinished()) { - s.finishedAt = int(step.GetFinished()) - } - if s.startedAt == 0 || s.startedAt > int(step.GetStarted()) { - s.startedAt = int(step.GetStarted()) +// nodeFromService returns a new node from a service +func nodeFromService(nodeID int, service *library.Service) *node { + return &node{ + ID: nodeID, + Cluster: ServiceCluster, + Name: service.GetName(), + Stage: nil, + Steps: []*library.Step{}, + Status: service.GetStatus(), + StartedAt: int(service.GetStarted()), + FinishedAt: int(service.GetFinished()), } } -func stageStatus(s *stg) string { - status := constants.StatusPending +// GetOverallStatus determines the "status" for a stage based on the steps within it. +// this could potentially get complicated with ruleset logic (continue/detach) +func (s *stg) GetOverallStatus() string { if s.running > 0 { - status = constants.StatusRunning - } else if s.failure > 0 { - status = constants.StatusFailure - } else if s.success > 0 { - status = constants.StatusSuccess - } else if s.killed > 0 { - status = constants.StatusKilled + return constants.StatusRunning } - return status + + if s.failure > 0 { + return constants.StatusFailure + } + + if s.success > 0 { + return constants.StatusSuccess + } + + if s.killed > 0 { + return constants.StatusKilled + } + + return constants.StatusPending } From a37df7eda296fad1dcb96464b2208f6fd37bd1a5 Mon Sep 17 00:00:00 2001 From: davidvader Date: Mon, 16 Oct 2023 16:49:23 -0500 Subject: [PATCH 14/36] fix: account for canceled and skipped --- api/build/graph.go | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/api/build/graph.go b/api/build/graph.go index b7a724c4a..a665cf4f0 100644 --- a/api/build/graph.go +++ b/api/build/graph.go @@ -71,6 +71,8 @@ type stg struct { running int failure int killed int + canceled int + skipped int startedAt int finishedAt int } @@ -138,7 +140,7 @@ func GetBuildGraph(c *gin.Context) { "user": u.GetName(), }) - baseErr := "unable to construct graph" + baseErr := "unable to retrieve graph" // update engine logger with API metadata // @@ -310,6 +312,8 @@ func GetBuildGraph(c *gin.Context) { running: 0, failure: 0, killed: 0, + canceled: 0, + skipped: 0, startedAt: 0, finishedAt: 0, } @@ -328,6 +332,10 @@ func GetBuildGraph(c *gin.Context) { s.failure++ case constants.StatusKilled: s.killed++ + case constants.StatusCanceled: + s.canceled++ + case constants.StatusSkipped: + s.skipped++ default: } @@ -556,5 +564,13 @@ func (s *stg) GetOverallStatus() string { return constants.StatusKilled } + if s.skipped > 0 { + return constants.StatusKilled + } + + if s.canceled > 0 { + return constants.StatusFailure + } + return constants.StatusPending } From 1c06bd8152fdfb030e59f08ac5275d206f1ab137 Mon Sep 17 00:00:00 2001 From: dave vader <48764154+plyr4@users.noreply.github.com> Date: Fri, 27 Oct 2023 14:12:12 -0500 Subject: [PATCH 15/36] fix: compile with commit Co-authored-by: Easton Crupper <65553218+ecrupper@users.noreply.github.com> --- api/build/graph.go | 1 + 1 file changed, 1 insertion(+) diff --git a/api/build/graph.go b/api/build/graph.go index a665cf4f0..d40cbf80d 100644 --- a/api/build/graph.go +++ b/api/build/graph.go @@ -261,6 +261,7 @@ func GetBuildGraph(c *gin.Context) { Duplicate(). WithBuild(b). WithFiles(files). + WithCommit(b.GetCommit()) WithMetadata(m). WithRepo(r). WithUser(u). From 9b970300d924b4d6d95451baab9ed1b1ec884e05 Mon Sep 17 00:00:00 2001 From: davidvader Date: Fri, 27 Oct 2023 14:36:02 -0500 Subject: [PATCH 16/36] fix: typo --- api/build/graph.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/build/graph.go b/api/build/graph.go index d40cbf80d..075fd8d99 100644 --- a/api/build/graph.go +++ b/api/build/graph.go @@ -261,7 +261,7 @@ func GetBuildGraph(c *gin.Context) { Duplicate(). WithBuild(b). WithFiles(files). - WithCommit(b.GetCommit()) + WithCommit(b.GetCommit()). WithMetadata(m). WithRepo(r). WithUser(u). From e87e10a7eb82f9daf9580c5b0ea21877b8a53acc Mon Sep 17 00:00:00 2001 From: davidvader Date: Fri, 27 Oct 2023 14:59:48 -0500 Subject: [PATCH 17/36] fix: add Graph to api spec --- api/build/graph.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/api/build/graph.go b/api/build/graph.go index 075fd8d99..1a94b7648 100644 --- a/api/build/graph.go +++ b/api/build/graph.go @@ -27,11 +27,13 @@ import ( "github.com/sirupsen/logrus" ) -// graph contains nodes, and relationships between nodes, or edges. +// Graph contains nodes, and relationships between nodes, or edges. // // a node is a pipeline stage and its relevant steps. // an edge is a relationship between nodes, defined by the 'needs' tag. -type graph struct { +// +// swagger:model Graph +type Graph struct { BuildID int64 `json:"build_id"` Nodes map[int]*node `json:"nodes"` Edges []*edge `json:"edges"` @@ -509,7 +511,7 @@ func GetBuildGraph(c *gin.Context) { } // construct the response - graph := graph{ + graph := Graph{ BuildID: b.GetID(), Nodes: nodes, Edges: edges, From f2ec456279bbd7be30b77e28b8883716b3036d74 Mon Sep 17 00:00:00 2001 From: davidvader Date: Mon, 30 Oct 2023 12:02:00 -0500 Subject: [PATCH 18/36] fix: reorganize imports --- api/build/graph.go | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/api/build/graph.go b/api/build/graph.go index 1a94b7648..01cf36ca4 100644 --- a/api/build/graph.go +++ b/api/build/graph.go @@ -9,21 +9,19 @@ import ( "net/http" "strings" + "github.com/gin-gonic/gin" "github.com/go-vela/server/compiler" "github.com/go-vela/server/database" + "github.com/go-vela/server/router/middleware/build" "github.com/go-vela/server/router/middleware/org" + "github.com/go-vela/server/router/middleware/repo" "github.com/go-vela/server/router/middleware/user" "github.com/go-vela/server/scm" + "github.com/go-vela/server/util" "github.com/go-vela/types" "github.com/go-vela/types/constants" "github.com/go-vela/types/library" "github.com/go-vela/types/pipeline" - - "github.com/go-vela/server/router/middleware/build" - "github.com/go-vela/server/router/middleware/repo" - "github.com/go-vela/server/util" - - "github.com/gin-gonic/gin" "github.com/sirupsen/logrus" ) From e7b5e8dce7093e9f408b9aad3944c51b2bb7b5f5 Mon Sep 17 00:00:00 2001 From: davidvader Date: Mon, 30 Oct 2023 12:10:17 -0500 Subject: [PATCH 19/36] enhance: use constant for complexity limit --- api/build/graph.go | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/api/build/graph.go b/api/build/graph.go index 01cf36ca4..899331da8 100644 --- a/api/build/graph.go +++ b/api/build/graph.go @@ -79,9 +79,10 @@ type stg struct { const ( // clusters determine graph orientation - BuiltInCluster = 2 - PipelineCluster = 1 - ServiceCluster = 0 + BuiltInCluster = 2 + PipelineCluster = 1 + ServiceCluster = 0 + GraphComplexityLimit = 1000 ) // swagger:operation GET /api/v1/repos/{org}/{repo}/builds/{build}/graph builds GetBuildGraph @@ -496,10 +497,8 @@ func GetBuildGraph(c *gin.Context) { } } - // for loop over edges, and collapse same parent edge - // todo: move this check above the processing ? - if len(nodes) > 5000 { - c.JSON(http.StatusInternalServerError, "too many nodes on this graph") + if len(nodes) > GraphComplexityLimit || len(nodes) > GraphComplexityLimit { + c.JSON(http.StatusInternalServerError, "too many nodes or edges on this graph") return } From 8fd12e2aa361d5ed0fe653d85109e9a03d040449 Mon Sep 17 00:00:00 2001 From: davidvader Date: Mon, 30 Oct 2023 12:26:27 -0500 Subject: [PATCH 20/36] enhance: check complexity prior to processing --- api/build/graph.go | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/api/build/graph.go b/api/build/graph.go index 899331da8..f7f5a6a53 100644 --- a/api/build/graph.go +++ b/api/build/graph.go @@ -82,7 +82,7 @@ const ( BuiltInCluster = 2 PipelineCluster = 1 ServiceCluster = 0 - GraphComplexityLimit = 1000 + GraphComplexityLimit = 1000 // arbitrary value to limit render complexity ) // swagger:operation GET /api/v1/repos/{org}/{repo}/builds/{build}/graph builds GetBuildGraph @@ -287,6 +287,14 @@ func GetBuildGraph(c *gin.Context) { return } + // this is a simple check + // but it will save on processing a massive build that should not be rendered + complexity := len(steps) + len(p.Stages) + len(services) + if complexity > GraphComplexityLimit { + c.JSON(http.StatusInternalServerError, "build is too complex, too many resources") + return + } + logger.Info("generating build graph") // create nodes from pipeline stages @@ -422,7 +430,7 @@ func GetBuildGraph(c *gin.Context) { // loop over all nodes and create edges based on 'needs' for _, destinationNode := range nodes { - // if theres no stage, skip because the edge is already created? + // if theres no stage, skip because the edge is already created if destinationNode.Stage == nil { continue } @@ -497,13 +505,9 @@ func GetBuildGraph(c *gin.Context) { } } - if len(nodes) > GraphComplexityLimit || len(nodes) > GraphComplexityLimit { - c.JSON(http.StatusInternalServerError, "too many nodes or edges on this graph") - return - } - - if len(edges) > 5000 { - c.JSON(http.StatusInternalServerError, "too many edges on this graph") + // validate the generated graph's complexity is beneath the limit + if len(nodes)+len(edges) > GraphComplexityLimit { + c.JSON(http.StatusInternalServerError, "graph is too complex, too many nodes and edges") return } From f8e4e3f3a7da9df9dbf7c0b365e586c6f8474e2f Mon Sep 17 00:00:00 2001 From: davidvader Date: Mon, 30 Oct 2023 12:27:53 -0500 Subject: [PATCH 21/36] fix: combine builtin node check --- api/build/graph.go | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/api/build/graph.go b/api/build/graph.go index f7f5a6a53..40d07f62f 100644 --- a/api/build/graph.go +++ b/api/build/graph.go @@ -456,11 +456,8 @@ func GetBuildGraph(c *gin.Context) { } // skip normal processing for built-in nodes - if destinationNode.Cluster == BuiltInCluster { - continue - } - - if sourceNode.Cluster == BuiltInCluster { + if destinationNode.Cluster == BuiltInCluster || + sourceNode.Cluster == BuiltInCluster { continue } From b1754b52beee24e886d4859a43222a7db25618ee Mon Sep 17 00:00:00 2001 From: davidvader Date: Wed, 1 Nov 2023 09:08:42 -0500 Subject: [PATCH 22/36] fix: account for all statuses --- api/build/graph.go | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/api/build/graph.go b/api/build/graph.go index 40d07f62f..6f110a88a 100644 --- a/api/build/graph.go +++ b/api/build/graph.go @@ -73,6 +73,7 @@ type stg struct { killed int canceled int skipped int + errored int startedAt int finishedAt int } @@ -324,6 +325,7 @@ func GetBuildGraph(c *gin.Context) { killed: 0, canceled: 0, skipped: 0, + errored: 0, startedAt: 0, finishedAt: 0, } @@ -346,6 +348,8 @@ func GetBuildGraph(c *gin.Context) { s.canceled++ case constants.StatusSkipped: s.skipped++ + case constants.StatusError: + s.errored++ default: } @@ -566,11 +570,15 @@ func (s *stg) GetOverallStatus() string { } if s.skipped > 0 { - return constants.StatusKilled + return constants.StatusSkipped } if s.canceled > 0 { - return constants.StatusFailure + return constants.StatusCanceled + } + + if s.errored > 0 { + return constants.StatusError } return constants.StatusPending From 51f804e3bca817393b5423db933daeceb0e149d8 Mon Sep 17 00:00:00 2001 From: davidvader Date: Wed, 1 Nov 2023 09:11:01 -0500 Subject: [PATCH 23/36] chore: revert gitignore --- .gitignore | 1 - 1 file changed, 1 deletion(-) diff --git a/.gitignore b/.gitignore index 29b233934..c557bc18b 100644 --- a/.gitignore +++ b/.gitignore @@ -48,7 +48,6 @@ api-spec.json # Edit at https://www.toptal.com/developers/gitignore?templates=visualstudiocode ### VisualStudioCode ### -.vscode/ .vscode/* !.vscode/settings.json !.vscode/tasks.json From 272722f1bdc44d694803ca4c73efc977e1d005d9 Mon Sep 17 00:00:00 2001 From: davidvader Date: Wed, 1 Nov 2023 09:12:20 -0500 Subject: [PATCH 24/36] chore: move comment --- api/build/graph.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/api/build/graph.go b/api/build/graph.go index 6f110a88a..ab80d5b09 100644 --- a/api/build/graph.go +++ b/api/build/graph.go @@ -134,6 +134,9 @@ func GetBuildGraph(c *gin.Context) { m := c.MustGet("metadata").(*types.Metadata) ctx := c.Request.Context() + // update engine logger with API metadata + // + // https://pkg.go.dev/github.com/sirupsen/logrus?tab=doc#Entry.WithFields entry := fmt.Sprintf("%s/%d", r.GetFullName(), b.GetNumber()) logger := logrus.WithFields(logrus.Fields{ "build": b.GetNumber(), @@ -144,9 +147,6 @@ func GetBuildGraph(c *gin.Context) { baseErr := "unable to retrieve graph" - // update engine logger with API metadata - // - // https://pkg.go.dev/github.com/sirupsen/logrus?tab=doc#Entry.WithFields logger.Infof("constructing graph for build %s", entry) // retrieve the steps for the build from the step table From 79a2a737987ff3890f8bc4c998905f352f36f5b1 Mon Sep 17 00:00:00 2001 From: davidvader Date: Wed, 1 Nov 2023 09:16:56 -0500 Subject: [PATCH 25/36] fix: use resource count returned from db --- api/build/graph.go | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/api/build/graph.go b/api/build/graph.go index ab80d5b09..58cadc356 100644 --- a/api/build/graph.go +++ b/api/build/graph.go @@ -155,7 +155,7 @@ func GetBuildGraph(c *gin.Context) { perPage := 100 for page > 0 { // retrieve build steps (per page) from the database - stepsPart, _, err := database.FromContext(c).ListStepsForBuild(b, nil, page, perPage) + stepsPart, stepsCount, err := database.FromContext(c).ListStepsForBuild(b, nil, page, perPage) if err != nil { retErr := fmt.Errorf("unable to retrieve steps for build %s: %w", entry, err) @@ -168,9 +168,7 @@ func GetBuildGraph(c *gin.Context) { steps = append(steps, stepsPart...) // assume no more pages exist if under 100 results are returned - // - // nolint: gomnd // ignore magic number - if len(stepsPart) < 100 { + if stepsCount < 100 { page = 0 } else { page++ @@ -189,7 +187,7 @@ func GetBuildGraph(c *gin.Context) { perPage = 100 for page > 0 { // retrieve build services (per page) from the database - servicesPart, _, err := database.FromContext(c).ListServicesForBuild(ctx, b, nil, page, perPage) + servicesPart, servicesCount, err := database.FromContext(c).ListServicesForBuild(ctx, b, nil, page, perPage) if err != nil { retErr := fmt.Errorf("unable to retrieve services for build %s: %w", entry, err) @@ -202,9 +200,7 @@ func GetBuildGraph(c *gin.Context) { services = append(services, servicesPart...) // assume no more pages exist if under 100 results are returned - // - // nolint: gomnd // ignore magic number - if len(servicesPart) < 100 { + if servicesCount < 100 { page = 0 } else { page++ From 1099f74946825e90cde3240c2b8d139567d86dca Mon Sep 17 00:00:00 2001 From: davidvader Date: Wed, 1 Nov 2023 09:22:13 -0500 Subject: [PATCH 26/36] fix: swagger headers --- api/build/graph.go | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/api/build/graph.go b/api/build/graph.go index 58cadc356..9bb123f04 100644 --- a/api/build/graph.go +++ b/api/build/graph.go @@ -115,9 +115,17 @@ const ( // '200': // description: Successfully retrieved graph for the build // schema: -// type: array +// type: json // items: // "$ref": "#/definitions/Graph" +// '401': +// description: Unable to retrieve graph for the build — unauthorized +// schema: +// "$ref": "#/definitions/Error" +// '404': +// description: Unable to retrieve graph for the build — not found +// schema: +// "$ref": "#/definitions/Error" // '500': // description: Unable to retrieve graph for the build // schema: From e60b37779e907096c1dc819eb0f6804b9d70eca4 Mon Sep 17 00:00:00 2001 From: davidvader Date: Wed, 1 Nov 2023 09:29:40 -0500 Subject: [PATCH 27/36] fix: linting --- api/build/graph.go | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/api/build/graph.go b/api/build/graph.go index 9bb123f04..cf30c4dd0 100644 --- a/api/build/graph.go +++ b/api/build/graph.go @@ -1,6 +1,4 @@ -// Copyright (c) 2021 Target Brands, Inc. All rights reserved. -// -// Use of this source code is governed by the LICENSE file in this repository. +// SPDX-License-Identifier: Apache-2.0 package build @@ -63,7 +61,7 @@ type edge struct { Status string `json:"status"` } -// stg represents a stage's steps and some metadata for producing node/edge information +// stg represents a stage's steps and some metadata for producing node/edge information. type stg struct { steps []*library.Step // used for tracking stage status @@ -79,11 +77,11 @@ type stg struct { } const ( - // clusters determine graph orientation + // clusters determine graph orientation. BuiltInCluster = 2 PipelineCluster = 1 ServiceCluster = 0 - GraphComplexityLimit = 1000 // arbitrary value to limit render complexity + GraphComplexityLimit = 1000 // arbitrary value to limit render complexity. ) // swagger:operation GET /api/v1/repos/{org}/{repo}/builds/{build}/graph builds GetBuildGraph @@ -133,6 +131,8 @@ const ( // GetBuildGraph represents the API handler to capture a // directed a-cyclical graph for a build from the configured backend. +// +//nolint:funlen,goconst // ignore function length and constants func GetBuildGraph(c *gin.Context) { // capture middleware values b := build.Retrieve(c) @@ -161,6 +161,7 @@ func GetBuildGraph(c *gin.Context) { steps := []*library.Step{} page := 1 perPage := 100 + for page > 0 { // retrieve build steps (per page) from the database stepsPart, stepsCount, err := database.FromContext(c).ListStepsForBuild(b, nil, page, perPage) @@ -193,6 +194,7 @@ func GetBuildGraph(c *gin.Context) { services := []*library.Service{} page = 1 perPage = 100 + for page > 0 { // retrieve build services (per page) from the database servicesPart, servicesCount, err := database.FromContext(c).ListServicesForBuild(ctx, b, nil, page, perPage) @@ -251,7 +253,7 @@ func GetBuildGraph(c *gin.Context) { // send API call to capture list of files changed for the commit files, err = scm.FromContext(c).Changeset(ctx, u, r, b.GetCommit()) if err != nil { - retErr := fmt.Errorf("%s: failed to get changeset for %s: %v", baseErr, r.GetFullName(), err) + retErr := fmt.Errorf("%s: failed to get changeset for %s: %w", baseErr, r.GetFullName(), err) util.HandleError(c, http.StatusInternalServerError, retErr) @@ -274,11 +276,11 @@ func GetBuildGraph(c *gin.Context) { Compile(config) if err != nil { // format the error message with extra information - err = fmt.Errorf("unable to compile pipeline configuration for %s: %v", r.GetFullName(), err) + err = fmt.Errorf("unable to compile pipeline configuration for %s: %w", r.GetFullName(), err) logger.Error(err.Error()) - retErr := fmt.Errorf("%s: %v", baseErr, err) + retErr := fmt.Errorf("%s: %w", baseErr, err) util.HandleError(c, http.StatusInternalServerError, retErr) @@ -313,6 +315,7 @@ func GetBuildGraph(c *gin.Context) { // initialize a map for grouping steps by stage name // and tracking stage information stageMap := map[string]*stg{} + for _, step := range steps { name := step.GetStage() if len(name) == 0 { @@ -361,6 +364,7 @@ func GetBuildGraph(c *gin.Context) { if s.finishedAt == 0 || s.finishedAt < int(step.GetFinished()) { s.finishedAt = int(step.GetFinished()) } + if s.startedAt == 0 || s.startedAt > int(step.GetStarted()) { s.startedAt = int(step.GetStarted()) } @@ -535,8 +539,8 @@ func nodeFromStage(nodeID, cluster int, stage *pipeline.Stage, s *stg) *node { Stage: stage, Steps: s.steps, Status: s.GetOverallStatus(), - StartedAt: int(s.startedAt), - FinishedAt: int(s.finishedAt), + StartedAt: s.startedAt, + FinishedAt: s.finishedAt, } } From 7038f73e30f984f2f5e84913c92ceea04a4fca0a Mon Sep 17 00:00:00 2001 From: davidvader Date: Wed, 1 Nov 2023 09:36:14 -0500 Subject: [PATCH 28/36] chore: revert docker-compose --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 32b2764f7..640497c32 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -27,7 +27,6 @@ services: QUEUE_DRIVER: redis QUEUE_ADDR: 'redis://redis:6379' QUEUE_PRIVATE_KEY: 'tCIevHOBq6DdN5SSBtteXUusjjd0fOqzk2eyi0DMq04NewmShNKQeUbbp3vkvIckb4pCxc+vxUo+mYf/vzOaSg==' - QUEUE_PUBLIC_KEY: 'DXsJkoTSkHlG26d75LyHJG+KQsXPr8VKPpmH/78zmko=' SCM_DRIVER: github SCM_CONTEXT: 'continuous-integration/vela' SECRET_VAULT: 'true' @@ -39,6 +38,7 @@ services: # comment the line below to use registration flow VELA_SECRET: 'zB7mrKDTZqNeNTD8z47yG4DHywspAh' VELA_SERVER_PRIVATE_KEY: 'F534FF2A080E45F38E05DC70752E6787' + QUEUE_PUBLIC_KEY: 'DXsJkoTSkHlG26d75LyHJG+KQsXPr8VKPpmH/78zmko=' VELA_USER_REFRESH_TOKEN_DURATION: 90m VELA_USER_ACCESS_TOKEN_DURATION: 60m VELA_WORKER_AUTH_TOKEN_DURATION: 3m From badc5c24823d1c37d3a8c5a5586161bb0d31bdb8 Mon Sep 17 00:00:00 2001 From: davidvader Date: Wed, 1 Nov 2023 09:36:41 -0500 Subject: [PATCH 29/36] chore: revert docker-compose --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 640497c32..349d0d34b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -37,8 +37,8 @@ services: VELA_LOG_LEVEL: trace # comment the line below to use registration flow VELA_SECRET: 'zB7mrKDTZqNeNTD8z47yG4DHywspAh' - VELA_SERVER_PRIVATE_KEY: 'F534FF2A080E45F38E05DC70752E6787' QUEUE_PUBLIC_KEY: 'DXsJkoTSkHlG26d75LyHJG+KQsXPr8VKPpmH/78zmko=' + VELA_SERVER_PRIVATE_KEY: 'F534FF2A080E45F38E05DC70752E6787' VELA_USER_REFRESH_TOKEN_DURATION: 90m VELA_USER_ACCESS_TOKEN_DURATION: 60m VELA_WORKER_AUTH_TOKEN_DURATION: 3m From 551fc8afc6d007bc4abe3e24689228d94cd01b5a Mon Sep 17 00:00:00 2001 From: davidvader Date: Wed, 1 Nov 2023 09:49:45 -0500 Subject: [PATCH 30/36] fix: api-spec schema --- api/build/graph.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/api/build/graph.go b/api/build/graph.go index cf30c4dd0..e853eeb33 100644 --- a/api/build/graph.go +++ b/api/build/graph.go @@ -112,10 +112,9 @@ const ( // responses: // '200': // description: Successfully retrieved graph for the build +// type: json // schema: -// type: json -// items: -// "$ref": "#/definitions/Graph" +// "$ref": "#/definitions/Graph" // '401': // description: Unable to retrieve graph for the build — unauthorized // schema: From 63d01b34546678172795876893e9e84e587a353d Mon Sep 17 00:00:00 2001 From: davidvader Date: Wed, 1 Nov 2023 09:54:56 -0500 Subject: [PATCH 31/36] fix: linting --- api/build/graph.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/api/build/graph.go b/api/build/graph.go index e853eeb33..00dd0db5c 100644 --- a/api/build/graph.go +++ b/api/build/graph.go @@ -185,7 +185,9 @@ func GetBuildGraph(c *gin.Context) { if len(steps) == 0 { retErr := fmt.Errorf("no steps found for build %s", entry) + util.HandleError(c, http.StatusNotFound, retErr) + return } From c0ac3df0486bb39977d44574f206d0324260164a Mon Sep 17 00:00:00 2001 From: davidvader Date: Wed, 1 Nov 2023 10:11:36 -0500 Subject: [PATCH 32/36] fix: use perPage var over hardcoded variable --- api/build/graph.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/api/build/graph.go b/api/build/graph.go index 00dd0db5c..c37e69f72 100644 --- a/api/build/graph.go +++ b/api/build/graph.go @@ -176,7 +176,7 @@ func GetBuildGraph(c *gin.Context) { steps = append(steps, stepsPart...) // assume no more pages exist if under 100 results are returned - if stepsCount < 100 { + if int(stepsCount) < perPage { page = 0 } else { page++ @@ -211,7 +211,7 @@ func GetBuildGraph(c *gin.Context) { services = append(services, servicesPart...) // assume no more pages exist if under 100 results are returned - if servicesCount < 100 { + if int(servicesCount) < perPage { page = 0 } else { page++ From f589dce4af50aeff6a507510b6212d57e46007f3 Mon Sep 17 00:00:00 2001 From: davidvader Date: Wed, 1 Nov 2023 15:38:46 -0500 Subject: [PATCH 33/36] enhance: resource fetch optimization --- api/build/graph.go | 130 ++++++++++++++++++++++++--------------------- 1 file changed, 68 insertions(+), 62 deletions(-) diff --git a/api/build/graph.go b/api/build/graph.go index c37e69f72..4af1b7730 100644 --- a/api/build/graph.go +++ b/api/build/graph.go @@ -156,68 +156,6 @@ func GetBuildGraph(c *gin.Context) { logger.Infof("constructing graph for build %s", entry) - // retrieve the steps for the build from the step table - steps := []*library.Step{} - page := 1 - perPage := 100 - - for page > 0 { - // retrieve build steps (per page) from the database - stepsPart, stepsCount, err := database.FromContext(c).ListStepsForBuild(b, nil, page, perPage) - if err != nil { - retErr := fmt.Errorf("unable to retrieve steps for build %s: %w", entry, err) - - util.HandleError(c, http.StatusNotFound, retErr) - - return - } - - // add page of steps to list steps - steps = append(steps, stepsPart...) - - // assume no more pages exist if under 100 results are returned - if int(stepsCount) < perPage { - page = 0 - } else { - page++ - } - } - - if len(steps) == 0 { - retErr := fmt.Errorf("no steps found for build %s", entry) - - util.HandleError(c, http.StatusNotFound, retErr) - - return - } - - // retrieve the services for the build from the service table - services := []*library.Service{} - page = 1 - perPage = 100 - - for page > 0 { - // retrieve build services (per page) from the database - servicesPart, servicesCount, err := database.FromContext(c).ListServicesForBuild(ctx, b, nil, page, perPage) - if err != nil { - retErr := fmt.Errorf("unable to retrieve services for build %s: %w", entry, err) - - util.HandleError(c, http.StatusNotFound, retErr) - - return - } - - // add page of services to list services - services = append(services, servicesPart...) - - // assume no more pages exist if under 100 results are returned - if int(servicesCount) < perPage { - page = 0 - } else { - page++ - } - } - logger.Info("retrieving pipeline configuration") var config []byte @@ -295,6 +233,74 @@ func GetBuildGraph(c *gin.Context) { return } + // retrieve the steps for the build from the step table + steps := []*library.Step{} + page := 1 + perPage := 100 + + // only fetch steps when necessary + if len(p.Stages) > 0 || len(p.Steps) > 0 { + for page > 0 { + // retrieve build steps (per page) from the database + stepsPart, stepsCount, err := database.FromContext(c).ListStepsForBuild(b, nil, page, perPage) + if err != nil { + retErr := fmt.Errorf("unable to retrieve steps for build %s: %w", entry, err) + + util.HandleError(c, http.StatusNotFound, retErr) + + return + } + + // add page of steps to list steps + steps = append(steps, stepsPart...) + + // assume no more pages exist if under 100 results are returned + if int(stepsCount) < perPage { + page = 0 + } else { + page++ + } + } + } + + if len(steps) == 0 { + retErr := fmt.Errorf("no steps found for build %s", entry) + + util.HandleError(c, http.StatusNotFound, retErr) + + return + } + + // retrieve the services for the build from the service table + services := []*library.Service{} + page = 1 + perPage = 100 + + // only fetch services when necessary + if len(p.Services) > 0 { + for page > 0 { + // retrieve build services (per page) from the database + servicesPart, servicesCount, err := database.FromContext(c).ListServicesForBuild(ctx, b, nil, page, perPage) + if err != nil { + retErr := fmt.Errorf("unable to retrieve services for build %s: %w", entry, err) + + util.HandleError(c, http.StatusNotFound, retErr) + + return + } + + // add page of services to list services + services = append(services, servicesPart...) + + // assume no more pages exist if under 100 results are returned + if int(servicesCount) < perPage { + page = 0 + } else { + page++ + } + } + } + // this is a simple check // but it will save on processing a massive build that should not be rendered complexity := len(steps) + len(p.Stages) + len(services) From aee68c2a158a815ff189c01d3c01db394a50986d Mon Sep 17 00:00:00 2001 From: davidvader Date: Thu, 2 Nov 2023 15:25:39 -0500 Subject: [PATCH 34/36] enhance: ruleset and status clarity --- api/build/graph.go | 34 ++++++++++++++++++++++++++-------- 1 file changed, 26 insertions(+), 8 deletions(-) diff --git a/api/build/graph.go b/api/build/graph.go index 4af1b7730..27e7bd16d 100644 --- a/api/build/graph.go +++ b/api/build/graph.go @@ -323,6 +323,19 @@ func GetBuildGraph(c *gin.Context) { // and tracking stage information stageMap := map[string]*stg{} + // build a map for step_id to pipeline step + stepMap := map[int]*pipeline.Container{} + + for _, pStep := range p.Steps { + stepMap[pStep.Number] = pStep + } + + for _, pStage := range p.Stages { + for _, pStep := range pStage.Steps { + stepMap[pStep.Number] = pStep + } + } + for _, step := range steps { name := step.GetStage() if len(name) == 0 { @@ -355,7 +368,12 @@ func GetBuildGraph(c *gin.Context) { case constants.StatusSuccess: s.success++ case constants.StatusFailure: - s.failure++ + stp, ok := stepMap[step.GetNumber()] + if ok && stp.Ruleset.Continue { + s.success++ + } else { + s.failure++ + } case constants.StatusKilled: s.killed++ case constants.StatusCanceled: @@ -576,11 +594,11 @@ func (s *stg) GetOverallStatus() string { return constants.StatusFailure } - if s.success > 0 { - return constants.StatusSuccess + if s.errored > 0 { + return constants.StatusError } - if s.killed > 0 { + if s.killed >= len(s.steps) { return constants.StatusKilled } @@ -588,12 +606,12 @@ func (s *stg) GetOverallStatus() string { return constants.StatusSkipped } - if s.canceled > 0 { - return constants.StatusCanceled + if s.success > 0 { + return constants.StatusSuccess } - if s.errored > 0 { - return constants.StatusError + if s.canceled > 0 { + return constants.StatusCanceled } return constants.StatusPending From 06f1ddf1d321a5e3b385429cfc54b01ef6c5e529 Mon Sep 17 00:00:00 2001 From: davidvader Date: Thu, 2 Nov 2023 15:27:10 -0500 Subject: [PATCH 35/36] fix: comments --- api/build/graph.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/api/build/graph.go b/api/build/graph.go index 27e7bd16d..096b712f4 100644 --- a/api/build/graph.go +++ b/api/build/graph.go @@ -555,7 +555,7 @@ func GetBuildGraph(c *gin.Context) { c.JSON(http.StatusOK, graph) } -// nodeFromStage returns a new node from a stage +// nodeFromStage returns a new node from a stage. func nodeFromStage(nodeID, cluster int, stage *pipeline.Stage, s *stg) *node { return &node{ ID: nodeID, @@ -569,7 +569,7 @@ func nodeFromStage(nodeID, cluster int, stage *pipeline.Stage, s *stg) *node { } } -// nodeFromService returns a new node from a service +// nodeFromService returns a new node from a service. func nodeFromService(nodeID int, service *library.Service) *node { return &node{ ID: nodeID, @@ -584,7 +584,7 @@ func nodeFromService(nodeID int, service *library.Service) *node { } // GetOverallStatus determines the "status" for a stage based on the steps within it. -// this could potentially get complicated with ruleset logic (continue/detach) +// this could potentially get complicated with ruleset logic (continue/detach). func (s *stg) GetOverallStatus() string { if s.running > 0 { return constants.StatusRunning From b9bf5da97cee90252877eea3866fc33864a706c9 Mon Sep 17 00:00:00 2001 From: davidvader Date: Thu, 2 Nov 2023 15:29:44 -0500 Subject: [PATCH 36/36] chore: remove unused initializations --- api/build/graph.go | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/api/build/graph.go b/api/build/graph.go index 096b712f4..c93cefa90 100644 --- a/api/build/graph.go +++ b/api/build/graph.go @@ -345,16 +345,7 @@ func GetBuildGraph(c *gin.Context) { // initialize a stage tracker if _, ok := stageMap[name]; !ok { stageMap[name] = &stg{ - steps: []*library.Step{}, - success: 0, - running: 0, - failure: 0, - killed: 0, - canceled: 0, - skipped: 0, - errored: 0, - startedAt: 0, - finishedAt: 0, + steps: []*library.Step{}, } }