diff --git a/cmd/app/cmd.go b/cmd/app/cmd.go index d8a3f2a5..16727cb4 100644 --- a/cmd/app/cmd.go +++ b/cmd/app/cmd.go @@ -9,6 +9,7 @@ import ( "flag" "io" "os" + "path/filepath" "github.com/gardener/docforge/pkg/api" "github.com/gardener/docforge/pkg/hugo" @@ -158,6 +159,13 @@ func NewOptions(f *cmdFlags) *Options { dryRunWriter = os.Stdout } + path, err := os.Getwd() + if err != nil { + //TODO: + } + + manifestAbsPath := filepath.Join(path, (f.documentationManifestPath)) + return &Options{ DestinationPath: f.destinationPath, FailFast: f.failFast, @@ -172,6 +180,7 @@ func NewOptions(f *cmdFlags) *Options { Resolve: f.resolve, GitHubInfoPath: f.ghInfoDestination, Hugo: hugoOptions, + ManifestAbsPath: manifestAbsPath, } } diff --git a/cmd/app/factory.go b/cmd/app/factory.go index 65dcffc7..45ca5f73 100644 --- a/cmd/app/factory.go +++ b/cmd/app/factory.go @@ -40,6 +40,7 @@ type Options struct { FailFast bool DestinationPath string ResourcesPath string + ManifestAbsPath string ResourceDownloadWorkersCount int RewriteEmbedded bool GitHubTokens map[string]string @@ -78,6 +79,7 @@ func NewReactor(ctx context.Context, options *Options, globalLinksCfg *api.Links DryRunWriter: dryRunWriters, Resolve: options.Resolve, GlobalLinksConfig: globalLinksCfg, + ManifestAbsPath: options.ManifestAbsPath, } if options.DryRunWriter != nil { o.Writer = dryRunWriters.GetWriter(options.DestinationPath) diff --git a/pkg/reactor/document_worker.go b/pkg/reactor/document_worker.go index 009bda32..555ab534 100644 --- a/pkg/reactor/document_worker.go +++ b/pkg/reactor/document_worker.go @@ -86,6 +86,7 @@ func (w *DocumentWorker) Work(ctx context.Context, task interface{}, wq jobs.Wor if len(task.Node.ContentSelectors) > 0 { for _, content := range task.Node.ContentSelectors { if sourceBlob, err = w.Reader.Read(ctx, content.Source); err != nil { + err = fmt.Errorf("unable to read source contents for %s: %w", content.Source, err) return jobs.NewWorkerError(err, 0) } if len(sourceBlob) == 0 { diff --git a/pkg/reactor/manifest_resolver.go b/pkg/reactor/manifest_resolver.go new file mode 100644 index 00000000..775fd084 --- /dev/null +++ b/pkg/reactor/manifest_resolver.go @@ -0,0 +1,263 @@ +package reactor + +import ( + "context" + "fmt" + "reflect" + "strings" + + "github.com/gardener/docforge/pkg/api" + "github.com/gardener/docforge/pkg/resourcehandlers" + "github.com/google/uuid" + "k8s.io/klog/v2" +) + +// ResolveManifest resolves the manifests into buildable model +func ResolveManifest(ctx context.Context, manifest *api.Documentation, rhRegistry resourcehandlers.Registry, manifestAbsPath string) error { + var ( + structure []*api.Node + err error + ) + if manifest.NodeSelector != nil { + node, err := resolveNodeSelector(ctx, rhRegistry, &api.Node{NodeSelector: manifest.NodeSelector}, make(map[string]bool), manifest.Links) + if err != nil { + return err + } + manifest.NodeSelector = nil + structure = node.Nodes + } + if structure == nil { + structure = manifest.Structure + } else { + // TODO: this should be rather merge than append + structure = append(manifest.Structure, structure...) + } + + if structure == nil { + return fmt.Errorf("document structure resolved to nil") + } + + if err = resolveStructure(ctx, rhRegistry, manifestAbsPath, structure, manifest.Links, make(map[string]bool)); err != nil { + return err + } + + manifest.Structure = structure + return nil +} + +func resolveStructure(ctx context.Context, rhRegistry resourcehandlers.Registry, manifestAbsPath string, nodes []*api.Node, globalLinksConfig *api.Links, visited map[string]bool) error { + for _, node := range nodes { + node.SetParentsDownwards() + if len(node.Source) > 0 { + nodeName, err := resolveNodeName(ctx, node, rhRegistry) + if err != nil { + return err + } + node.Name = nodeName + } + if node.NodeSelector != nil { + if manifestAbsPath == node.NodeSelector.Path { + klog.Warningf("circular dependency discovered, the node %s specifies the provided documentation manifest with path %s as a dependency: ", node.Name, manifestAbsPath) + node.NodeSelector = nil + continue + } + if visited[node.NodeSelector.Path] { + klog.Warning("circular dependency discovered:", buildCircularDepMessage(visited, node.NodeSelector)) + node.NodeSelector = nil + continue + } + visited[node.NodeSelector.Path] = true + newNode, err := resolveNodeSelector(ctx, rhRegistry, node, visited, globalLinksConfig) + if err != nil { + return err + } + if len(newNode.Nodes) > 0 { + node.Nodes = append(node.Nodes, newNode.Nodes...) + } + node.Links = mergeLinks(node.Links, newNode.Links) + node.NodeSelector = nil + } + if len(node.Nodes) > 0 { + if err := resolveStructure(ctx, rhRegistry, manifestAbsPath, node.Nodes, globalLinksConfig, visited); err != nil { + return err + } + visited = map[string]bool{} + } + node.SetParentsDownwards() + } + return nil +} + +func resolveNodeSelector(ctx context.Context, rhRegistry resourcehandlers.Registry, node *api.Node, visited map[string]bool, globalLinksConfig *api.Links) (*api.Node, error) { + newNode := &api.Node{} + handler := rhRegistry.Get(node.NodeSelector.Path) + if handler == nil { + return nil, fmt.Errorf("No suitable handler registered for path %s", node.NodeSelector.Path) + } + + moduleDocumentation, err := handler.ResolveDocumentation(ctx, node.NodeSelector.Path) + if err != nil { + err = fmt.Errorf("failed to resolve imported documentation manifest for node %s with path %s: %v", node.Name, node.NodeSelector.Path, err) + return nil, err + } + + if moduleDocumentation != nil { + newNode.Nodes = moduleDocumentation.Structure + if moduleLinks := moduleDocumentation.Links; moduleLinks != nil { + globalNode := &api.Node{ + Links: globalLinksConfig, + } + pruneModuleLinks(moduleLinks.Rewrites, node, getNodeRewrites) + pruneModuleLinks(moduleLinks.Rewrites, globalNode, getNodeRewrites) + if moduleLinks.Downloads != nil { + pruneModuleLinks(moduleLinks.Downloads.Renames, node, getNodeDownloadsRenamesKeys) + pruneModuleLinks(moduleLinks.Downloads.Renames, globalNode, getNodeDownloadsRenamesKeys) + pruneModuleLinks(moduleLinks.Downloads.Scope, node, getNodeDownloadsScopeKeys) + pruneModuleLinks(moduleLinks.Downloads.Scope, globalNode, getNodeDownloadsScopeKeys) + } + } + newNode.Links = moduleDocumentation.Links + if moduleDocumentation.NodeSelector != nil { + if visited[moduleDocumentation.NodeSelector.Path] { + klog.Warning("circular dependency discovered:", buildCircularDepMessage(visited, moduleDocumentation.NodeSelector)) + moduleDocumentation.NodeSelector = nil + return newNode, nil + } + visited[moduleDocumentation.NodeSelector.Path] = true + childNode := &api.Node{ + NodeSelector: moduleDocumentation.NodeSelector, + } + childNode.SetParent(node) + res, err := resolveNodeSelector(ctx, rhRegistry, childNode, visited, globalLinksConfig) + if err != nil { + return nil, err + } + for _, n := range res.Nodes { + n.SetParent(node) + n.SetParentsDownwards() + } + pruneChildNodesLinks(node, res.Nodes, globalLinksConfig) + newNode.Nodes = append(newNode.Nodes, res.Nodes...) + } + return newNode, nil + } + + nodes, err := handler.ResolveNodeSelector(ctx, node, node.NodeSelector.ExcludePaths, node.NodeSelector.ExcludeFrontMatter, node.NodeSelector.FrontMatter, node.NodeSelector.Depth) + if err != nil { + return nil, err + } + + newNode.Nodes = nodes + return newNode, nil +} + +func buildCircularDepMessage(visited map[string]bool, nodeSelector *api.NodeSelector) string { + var circularDependency string + for path := range visited { + circularDependency = circularDependency + path + " -> " + } + circularDependency = circularDependency + nodeSelector.Path + return circularDependency +} + +func resolveNodeName(ctx context.Context, node *api.Node, rhRegistry resourcehandlers.Registry) (string, error) { + name := node.Name + handler := rhRegistry.Get(node.Source) + if handler == nil { + return "", fmt.Errorf("No suitable handler registered for URL %s", node.Source) + } + if len(node.Name) == 0 { + name = "$name" + } + resourceName, ext := handler.ResourceName(node.Source) + id := uuid.New().String() + name = strings.ReplaceAll(name, "$name", resourceName) + name = strings.ReplaceAll(name, "$uuid", id) + name = strings.ReplaceAll(name, "$ext", fmt.Sprintf(".%s", ext)) + return name, nil +} + +func pruneModuleLinks(moduleLinks interface{}, node *api.Node, getParentLinks func(node *api.Node) map[string]struct{}) { + v := reflect.ValueOf(moduleLinks) + if v.Kind() != reflect.Map { + return + } + + for _, moduleLinkKey := range v.MapKeys() { + for parent := node; parent != nil; parent = parent.Parent() { + if parent.Links == nil { + continue + } + + if parentLinks := getParentLinks(parent); parentLinks != nil { + if _, ok := parentLinks[moduleLinkKey.String()]; ok { + klog.Warningf("Overriding module link %s", moduleLinkKey.String()) + v.SetMapIndex(moduleLinkKey, reflect.Value{}) + } + } + } + } +} + +func getNodeRewrites(node *api.Node) map[string]struct{} { + if node.Links == nil { + return nil + } + keys := make(map[string]struct{}) + for k := range node.Links.Rewrites { + keys[k] = struct{}{} + } + return keys +} + +func getNodeDownloadsRenamesKeys(node *api.Node) map[string]struct{} { + if node.Links == nil { + return nil + } + if node.Links.Downloads == nil { + return nil + } + + keys := make(map[string]struct{}) + for k := range node.Links.Downloads.Renames { + keys[k] = struct{}{} + } + return keys +} + +func getNodeDownloadsScopeKeys(node *api.Node) map[string]struct{} { + if node.Links == nil { + return nil + } + if node.Links.Downloads == nil { + return nil + } + + keys := make(map[string]struct{}) + for k := range node.Links.Downloads.Scope { + keys[k] = struct{}{} + } + return keys +} + +func pruneChildNodesLinks(parentNode *api.Node, nodes []*api.Node, globalLinksConfig *api.Links) { + for _, node := range nodes { + if node.Nodes != nil { + pruneChildNodesLinks(node, node.Nodes, globalLinksConfig) + } + + if node.Links != nil { + globalNode := &api.Node{ + Links: globalLinksConfig, + } + pruneModuleLinks(node.Links.Rewrites, parentNode, getNodeRewrites) + pruneModuleLinks(node.Links.Rewrites, globalNode, getNodeRewrites) + if node.Links.Downloads != nil { + pruneModuleLinks(node.Links.Downloads.Renames, parentNode, getNodeDownloadsRenamesKeys) + pruneModuleLinks(node.Links.Downloads.Renames, globalNode, getNodeDownloadsRenamesKeys) + pruneModuleLinks(node.Links.Downloads.Scope, parentNode, getNodeDownloadsScopeKeys) + pruneModuleLinks(node.Links.Downloads.Scope, globalNode, getNodeDownloadsScopeKeys) + } + } + } +} diff --git a/pkg/reactor/manifest_resolver_test.go b/pkg/reactor/manifest_resolver_test.go new file mode 100644 index 00000000..0426e615 --- /dev/null +++ b/pkg/reactor/manifest_resolver_test.go @@ -0,0 +1,582 @@ +package reactor + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/gardener/docforge/pkg/api" + "github.com/gardener/docforge/pkg/resourcehandlers" + "github.com/gardener/docforge/pkg/resourcehandlers/testhandler" + "github.com/stretchr/testify/assert" + "k8s.io/utils/pointer" +) + +const testPath string = "test-nodesSelector-path" +const testPath2 string = "test-nodesSelector-path-2" + +var defaultCtxWithTimeout, _ = context.WithTimeout(context.TODO(), 5*time.Second) +var testNodeSelector = api.NodeSelector{Path: testPath} +var testNodeSelector2 = api.NodeSelector{Path: testPath2} +var testNode = api.Node{Name: "testNodeName", Source: "testNodeSource"} +var testNode2 = api.Node{Name: "testNodeName2", Source: "testNodeSource2"} + +func TestResolveManifest(t *testing.T) { + type args struct { + ctx context.Context + resolveNodeSelectorFunc func(ctx context.Context, node *api.Node, excludePaths []string, + frontMatter map[string]interface{}, excludeFrontMatter map[string]interface{}, depth int32) ([]*api.Node, error) + manifestPath string + testDocumentation *api.Documentation + } + tests := []struct { + name string + description string + args args + wantErr bool + expectedDocumentation *api.Documentation + }{ + { + name: "returns_err_when_missing_nodeSelector_and_structure", + description: "has error when there are no nodes after NodeSelector resolving and there are no nodes defined in Documentation.Structure", + args: args{ + ctx: defaultCtxWithTimeout, + resolveNodeSelectorFunc: func(ctx context.Context, node *api.Node, excludePaths []string, + frontMatter map[string]interface{}, excludeFrontMatter map[string]interface{}, depth int32) ([]*api.Node, error) { + return []*api.Node{}, nil + }, + testDocumentation: &api.Documentation{ + NodeSelector: &testNodeSelector, + }, + }, + wantErr: true, + }, + { + name: "succeeds_to_append_resolved_nodeSelector_nodes_to_structure", + description: "should resolve documentation nodeSelector on root level and append nodes to Documentation.Structure", + args: args{ + ctx: defaultCtxWithTimeout, + resolveNodeSelectorFunc: func(ctx context.Context, node *api.Node, excludePaths []string, + frontMatter map[string]interface{}, excludeFrontMatter map[string]interface{}, depth int32) ([]*api.Node, error) { + return []*api.Node{&testNode}, nil + }, + testDocumentation: &api.Documentation{ + NodeSelector: &testNodeSelector, + }, + }, + wantErr: false, + expectedDocumentation: &api.Documentation{ + Structure: []*api.Node{&testNode}, + }, + }, + { + name: "succeeds_to_resolve_manifest", + description: "should resolve manifest and add the resolved nodesSelector nodes to existing structure", + args: args{ + ctx: defaultCtxWithTimeout, + resolveNodeSelectorFunc: func(ctx context.Context, node *api.Node, excludePaths []string, + frontMatter map[string]interface{}, excludeFrontMatter map[string]interface{}, depth int32) ([]*api.Node, error) { + return []*api.Node{&testNode}, nil + }, + testDocumentation: &api.Documentation{ + NodeSelector: &testNodeSelector, + Structure: []*api.Node{{Name: "existing"}}, + }, + }, + wantErr: false, + expectedDocumentation: &api.Documentation{ + Structure: []*api.Node{ + { + Name: "existing", + }, + &testNode, + }, + }, + }, + { + name: "resolves_child_node_nodeSelector", + description: "should resolve Node.NodeSelector nodes and append to the Node", + args: args{ + ctx: defaultCtxWithTimeout, + resolveNodeSelectorFunc: func(ctx context.Context, node *api.Node, excludePaths []string, + frontMatter map[string]interface{}, excludeFrontMatter map[string]interface{}, depth int32) ([]*api.Node, error) { + return []*api.Node{&testNode}, nil + }, + testDocumentation: &api.Documentation{ + Structure: []*api.Node{ + { + Name: "testNodeStructure", + NodeSelector: &testNodeSelector, + }, + }, + }, + }, + wantErr: false, + expectedDocumentation: &api.Documentation{ + Structure: []*api.Node{ + { + Name: "testNodeStructure", + Nodes: []*api.Node{ + &testNode, + }, + }, + }, + }, + }, + { + // TODO: this test case demonstrates design flaw where we generate anonmyous node + name: "resolve_module_on_root", + description: "resolve module specified on root level and append to structure", + args: args{ + ctx: defaultCtxWithTimeout, + resolveNodeSelectorFunc: func(ctx context.Context, node *api.Node, excludePaths []string, + frontMatter map[string]interface{}, excludeFrontMatter map[string]interface{}, depth int32) ([]*api.Node, error) { + if node.NodeSelector.Path == testNodeSelector.Path { + return []*api.Node{{NodeSelector: &api.NodeSelector{Path: "module.yaml"}}}, nil + } + return []*api.Node{{Name: "moduleNode"}}, nil + }, + testDocumentation: &api.Documentation{ + NodeSelector: &testNodeSelector, + Structure: []*api.Node{ + &testNode, + }, + }, + }, + wantErr: false, + expectedDocumentation: &api.Documentation{ + Structure: []*api.Node{ + &testNode, + { + Nodes: []*api.Node{{Name: "moduleNode"}}, + }, + }, + }, + }, + { + name: "break_recursive_module", + description: "breaks recursive import of modules for example if the documentation imports A that imports B it should stop resolvin if B imports A", + args: args{ + ctx: defaultCtxWithTimeout, + resolveNodeSelectorFunc: func(ctx context.Context, node *api.Node, excludePaths []string, + frontMatter map[string]interface{}, excludeFrontMatter map[string]interface{}, depth int32) ([]*api.Node, error) { + if node.NodeSelector.Path == testNodeSelector.Path { + return []*api.Node{{NodeSelector: &api.NodeSelector{Path: "moduleA.yaml"}}}, nil + } + if node.NodeSelector.Path == "moduleA.yaml" { + return []*api.Node{{NodeSelector: &testNodeSelector}}, nil + } + return []*api.Node{{Name: "resolvedNestedNode"}}, nil + }, + testDocumentation: &api.Documentation{NodeSelector: &testNodeSelector}, + }, + wantErr: false, + // TODO: circular dependency should clear empty nodes + expectedDocumentation: &api.Documentation{ + Structure: []*api.Node{ + { + Nodes: []*api.Node{ + { + Nodes: []*api.Node{ + {}, + }, + }, + }, + }, + }, + }, + }, + { + name: "succeeds_to_break_recursive_module", + args: args{ + ctx: defaultCtxWithTimeout, + resolveNodeSelectorFunc: func(ctx context.Context, node *api.Node, excludePaths []string, + frontMatter map[string]interface{}, excludeFrontMatter map[string]interface{}, depth int32) ([]*api.Node, error) { + if node.NodeSelector.Path == testNodeSelector.Path { + return []*api.Node{{NodeSelector: &api.NodeSelector{Path: "moduleA.yaml"}}}, nil + } + if node.NodeSelector.Path == "moduleA.yaml" { + return []*api.Node{{NodeSelector: &testNodeSelector}}, nil + } + return []*api.Node{{Name: "resolvedNestedNode"}}, nil + }, + testDocumentation: &api.Documentation{ + Structure: []*api.Node{ + &testNode, + { + Name: "moduleA", + NodeSelector: &testNodeSelector, + }, + }, + }, + }, + wantErr: false, + // TODO: circular dependency should clear empty nodes + expectedDocumentation: &api.Documentation{ + Structure: []*api.Node{ + &testNode, + { + Name: "moduleA", + Nodes: []*api.Node{ + { + Nodes: []*api.Node{ + {}, + }, + }, + }, + }, + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if tt.expectedDocumentation != nil { + for _, node := range tt.expectedDocumentation.Structure { + node.SetParentsDownwards() + } + } + rh := new(testhandler.TestResourceHandler).WithResolveNodeSelector(tt.args.resolveNodeSelectorFunc) + resourcehandlers.NewRegistry(rh) + if err := ResolveManifest(tt.args.ctx, tt.args.testDocumentation, resourcehandlers.NewRegistry(rh), tt.args.manifestPath); (err != nil) != tt.wantErr { + t.Errorf("ResolveManifest() error = %v, wantErr %v", err, tt.wantErr) + } + if tt.wantErr { + return + } + assert.Equal(t, tt.expectedDocumentation, tt.args.testDocumentation) + }) + } +} + +func Test_resolveNodeSelector(t *testing.T) { + type args struct { + ctx context.Context + rhRegistry resourcehandlers.Registry + node *api.Node + visited map[string]bool + globalLinksConfig *api.Links + } + tests := []struct { + name string + description string + args args + acceptFunc func(uri string) bool + resolveDocumentationFunc func(ctx context.Context, uri string) (*api.Documentation, error) + resolveNodeSelectorFunc func(ctx context.Context, node *api.Node, excludePaths []string, + frontMatter map[string]interface{}, excludeFrontMatter map[string]interface{}, depth int32) ([]*api.Node, error) + want *api.Node + wantErr bool + }{ + { + name: "missing_resource_handler", + description: "not suitable resource handler for path returns error", + args: args{ + node: &api.Node{ + NodeSelector: &testNodeSelector, + }, + visited: make(map[string]bool), + globalLinksConfig: &api.Links{}, + }, + acceptFunc: func(uri string) bool { + return !(uri == testNodeSelector.Path) + }, + + want: nil, + wantErr: true, + }, + { + name: "handler_resolve_documnetation_fails", + description: "the error when handler fails to resolve Documentation from NodeSelector`s path is propageted to the clien function", + args: args{ + node: &api.Node{ + NodeSelector: &testNodeSelector, + }, + visited: make(map[string]bool), + globalLinksConfig: &api.Links{}, + }, + resolveDocumentationFunc: func(ctx context.Context, uri string) (*api.Documentation, error) { + return nil, fmt.Errorf("error that should be thrown for this test case") + }, + want: nil, + wantErr: true, + }, + { + name: "succeeds_to_resolve_structure_with_nodeSelector_on_root", + description: "the test successfully resolves the api.Documentation.NodeSelector which path refers to another documentation structure", + args: args{ + node: &api.Node{ + NodeSelector: &testNodeSelector, + }, + visited: make(map[string]bool), + globalLinksConfig: &api.Links{}, + }, + resolveDocumentationFunc: func(ctx context.Context, uri string) (*api.Documentation, error) { + module := &api.Documentation{ + Structure: []*api.Node{ + &testNode, + }, + } + return module, nil + }, + want: &api.Node{ + Nodes: []*api.Node{ + &testNode, + }, + }, + wantErr: false, + }, + { + name: "succeeds_to_resolve_with_nodeSelector_and_strucutre_on_root", + description: "returns a node that combines nodes returned from the nodeSelector and refers to another documentation structure with nodeSelector and Structure nodes", + args: args{ + node: &api.Node{ + NodeSelector: &testNodeSelector, + }, + visited: make(map[string]bool), + globalLinksConfig: &api.Links{}, + }, + resolveNodeSelectorFunc: func(ctx context.Context, node *api.Node, excludePaths []string, frontMatter, excludeFrontMatter map[string]interface{}, depth int32) ([]*api.Node, error) { + if node.NodeSelector.Path == testNodeSelector2.Path { + return []*api.Node{&testNode2}, nil + } + return nil, nil + }, + resolveDocumentationFunc: func(ctx context.Context, uri string) (*api.Documentation, error) { + if uri == testPath { + module := &api.Documentation{ + NodeSelector: &testNodeSelector2, + Structure: []*api.Node{ + &testNode, + }, + } + return module, nil + } + return nil, nil + }, + want: &api.Node{ + Nodes: []*api.Node{ + &testNode, + &testNode2, + }, + }, + wantErr: false, + }, + { + name: "succeeds_to_merge_links_that_not_override_globalLinks", + description: "only return rewrite links that doesn't override the global Download links", + args: args{ + node: &api.Node{ + NodeSelector: &testNodeSelector, + }, + visited: make(map[string]bool), + globalLinksConfig: &api.Links{Rewrites: map[string]*api.LinkRewriteRule{ + "example.com/sources": { + Destination: pointer.StringPtr("A"), + }, + }}, + }, + resolveDocumentationFunc: func(ctx context.Context, uri string) (*api.Documentation, error) { + if uri == testPath { + module := &api.Documentation{ + Structure: []*api.Node{ + &testNode, + }, + Links: &api.Links{ + Rewrites: map[string]*api.LinkRewriteRule{ + "example.com/sources": { + Destination: pointer.StringPtr("shouldn't override A"), + }, + "example.com/blogs": { + Destination: pointer.StringPtr("should be added"), + }, + }, + }, + } + return module, nil + } + return nil, nil + }, + want: &api.Node{ + Nodes: []*api.Node{ + &testNode, + }, + Links: &api.Links{ + Rewrites: map[string]*api.LinkRewriteRule{ + "example.com/blogs": { + Destination: pointer.StringPtr("should be added"), + }, + }, + }, + }, + wantErr: false, + }, + { + name: "succeeds_to_merge_links_that_not_override_parent_links", + description: "only return rewrite links that doesn't override the parent or the global rewrite links ", + args: args{ + node: &api.Node{ + NodeSelector: &testNodeSelector, + Links: &api.Links{Rewrites: map[string]*api.LinkRewriteRule{ + "example.com/sources": { + Destination: pointer.StringPtr("parent node sources link rewrite"), + }, + "example.com/blogs": { + Destination: pointer.StringPtr("parent node blogs link rewrite"), + }, + }, + }, + }, + visited: make(map[string]bool), + globalLinksConfig: &api.Links{Rewrites: map[string]*api.LinkRewriteRule{ + "example.com/sources": { + Destination: pointer.StringPtr("sources global destination rewrite"), + }, + "example.com/news": { + Destination: pointer.StringPtr("news global destination rewrite"), + }, + }}, + }, + resolveDocumentationFunc: func(ctx context.Context, uri string) (*api.Documentation, error) { + if uri == testPath { + module := &api.Documentation{ + Structure: []*api.Node{ + &testNode, + }, + Links: &api.Links{ + Rewrites: map[string]*api.LinkRewriteRule{ + "example.com/sources": { + Destination: pointer.StringPtr("sources module destination rewrite"), + }, + "example.com/fish": { + Destination: pointer.StringPtr("fish module destination rewrite"), + }, + "example.com/news": { + Destination: pointer.StringPtr("news module destination rewrite"), + }, + "example.com/blogs": { + Destination: pointer.StringPtr("blogs module destination rewrite"), + }, + }, + }, + } + return module, nil + } + return nil, nil + }, + want: &api.Node{ + Nodes: []*api.Node{ + &testNode, + }, + Links: &api.Links{ + Rewrites: map[string]*api.LinkRewriteRule{ + "example.com/fish": { + Destination: pointer.StringPtr("fish module destination rewrite"), + }, + }, + }, + }, + wantErr: false, + }, + { + name: "succeeds_to_merge_download_links_that_not_override_parent_links", + description: "only return download links that doesn't override the parent or the global Download links", + args: args{ + node: &api.Node{ + NodeSelector: &testNodeSelector, + Links: &api.Links{ + Downloads: &api.Downloads{ + Renames: api.ResourceRenameRules{ + "example.com/images/image.png": "parent node download renames that exist in module", + "example.com/images/overridesglobal.png": "parent node download renames that overridesglobal and exist in module", + }, + Scope: map[string]api.ResourceRenameRules{ + "example.com/own/repo": { + "images/": "", + }, + }, + }, + }, + }, + visited: make(map[string]bool), + globalLinksConfig: &api.Links{ + Downloads: &api.Downloads{ + Renames: api.ResourceRenameRules{ + "example.com/images/overridesglobal.png": "global download renames that overridesglobal and exist in module", + "example.com/images/notexistonparent.png": "global download rename that doesn't exist on parent node", + }, + }, + }, + }, + resolveDocumentationFunc: func(ctx context.Context, uri string) (*api.Documentation, error) { + if uri == testPath { + module := &api.Documentation{ + Structure: []*api.Node{ + &testNode, + }, + Links: &api.Links{ + Downloads: &api.Downloads{ + Renames: api.ResourceRenameRules{ + "example.com/images/image.png": "parent node download renames that exist in module", + "example.com/images/overridesglobal.png": "parent node download renames that overridesglobal and exist in module", + "example.com/images/notexistonparent.png": "global download rename that doesn't exist on parent node", + "example.com/images/frommodule.png": "renames from module", + }, + Scope: map[string]api.ResourceRenameRules{ + "example.com/own/repo": { + "images/": "images/2.0", + }, + "example.com/notown/notrepo": { + "images/": "images/2.0", + }, + }, + }, + }, + } + return module, nil + } + return nil, nil + }, + want: &api.Node{ + Nodes: []*api.Node{ + &testNode, + }, + Links: &api.Links{ + Downloads: &api.Downloads{ + Renames: api.ResourceRenameRules{ + + "example.com/images/frommodule.png": "renames from module", + }, + Scope: map[string]api.ResourceRenameRules{ + "example.com/notown/notrepo": { + "images/": "images/2.0", + }, + }, + }, + }, + }, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if tt.want != nil { + for _, node := range tt.want.Nodes { + node.SetParentsDownwards() + } + } + rh := new(testhandler.TestResourceHandler).WithAccept(tt.acceptFunc).WithResolveDocumentation(tt.resolveDocumentationFunc).WithResolveNodeSelector(tt.resolveNodeSelectorFunc) + rhRegistry := resourcehandlers.NewRegistry(rh) + got, err := resolveNodeSelector(defaultCtxWithTimeout, rhRegistry, tt.args.node, tt.args.visited, tt.args.globalLinksConfig) + if (err != nil) != tt.wantErr { + t.Errorf("resolveNodeSelector() error = %v, wantErr %v", err, tt.wantErr) + return + } + + if !assert.Equal(t, tt.want, got) { + t.Errorf("resolveNodeSelector() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/pkg/reactor/reactor.go b/pkg/reactor/reactor.go index 08f31a6b..da916762 100644 --- a/pkg/reactor/reactor.go +++ b/pkg/reactor/reactor.go @@ -6,15 +6,11 @@ package reactor import ( "context" - "fmt" "io" "os" - "reflect" - "strings" "text/template" "github.com/gardener/docforge/pkg/processors" - "github.com/google/uuid" "k8s.io/klog/v2" "github.com/gardener/docforge/pkg/api" @@ -30,6 +26,7 @@ type Options struct { FailFast bool DestinationPath string ResourcesPath string + ManifestAbsPath string ResourceDownloadWorkersCount int RewriteEmbedded bool processors.Processor @@ -67,6 +64,7 @@ func NewReactor(o *Options) *Reactor { GitInfoController: gitInfoController, DryRunWriter: o.DryRunWriter, Resolve: o.Resolve, + manifestAbsPath: o.ManifestAbsPath, } return r } @@ -80,6 +78,7 @@ type Reactor struct { GitInfoController GitInfoController DryRunWriter writers.DryRunWriter Resolve bool + manifestAbsPath string } // Run starts build operation on documentation @@ -97,7 +96,7 @@ func (r *Reactor) Run(ctx context.Context, manifest *api.Documentation, dryRun b } }() - if err := r.ResolveManifest(ctx, manifest); err != nil { + if err := ResolveManifest(ctx, manifest, r.ResourceHandlers, r.manifestAbsPath); err != nil { return err } @@ -109,39 +108,6 @@ func (r *Reactor) Run(ctx context.Context, manifest *api.Documentation, dryRun b return nil } -// ResolveManifest resolves the manifests into buildable model -func (r *Reactor) ResolveManifest(ctx context.Context, manifest *api.Documentation) error { - var ( - structure []*api.Node - err error - ) - if manifest.NodeSelector != nil { - node, err := r.resolveNodeSelector(ctx, &api.Node{NodeSelector: manifest.NodeSelector}, manifest.Links) - if err != nil { - return err - } - manifest.NodeSelector = nil - structure = node.Nodes - } - if structure == nil { - structure = manifest.Structure - } else { - // TODO: this should be rather merge than append - structure = append(manifest.Structure, structure...) - } - - if structure == nil { - return fmt.Errorf("document structure resolved to nil") - } - - if err = r.resolveStructure(ctx, structure, manifest.Links); err != nil { - return err - } - - manifest.Structure = structure - return nil -} - func printResolved(ctx context.Context, manifest *api.Documentation, writer io.Writer) error { // for _, node := range manifest.Structure { // if links := resolveNodeLinks(node, manifest.Links); len(links) > 0 { @@ -163,195 +129,3 @@ func printResolved(ctx context.Context, manifest *api.Documentation, writer io.W writer.Write([]byte("\n\n")) return nil } - -// ResolveStructure resolves the following in a structure model: -// - Node name variables -// - NodeSelectors -// The resulting model is the actual flight plan for replicating resources. -func (r *Reactor) resolveStructure(ctx context.Context, nodes []*api.Node, globalLinksConfig *api.Links) error { - for _, node := range nodes { - node.SetParentsDownwards() - if len(node.Source) > 0 { - nodeName, err := r.resolveNodeName(ctx, node) - if err != nil { - return err - } - node.Name = nodeName - } - if node.NodeSelector != nil { - newNode, err := r.resolveNodeSelector(ctx, node, globalLinksConfig) - if err != nil { - return err - } - node.Nodes = append(node.Nodes, newNode.Nodes...) - node.Links = mergeLinks(node.Links, newNode.Links) - node.NodeSelector = nil - } - if len(node.Nodes) > 0 { - if err := r.resolveStructure(ctx, node.Nodes, globalLinksConfig); err != nil { - return err - } - } - node.SetParentsDownwards() - } - return nil -} - -func (r *Reactor) resolveNodeSelector(ctx context.Context, node *api.Node, globalLinksConfig *api.Links) (*api.Node, error) { - newNode := &api.Node{} - handler := r.ResourceHandlers.Get(node.NodeSelector.Path) - if handler == nil { - return nil, fmt.Errorf("No suitable handler registered for path %s", node.NodeSelector.Path) - } - - moduleDocumentation, err := handler.ResolveDocumentation(ctx, node.NodeSelector.Path) - if err != nil { - return nil, err - } - - if moduleDocumentation != nil { - newNode.Nodes = moduleDocumentation.Structure - if moduleLinks := moduleDocumentation.Links; moduleLinks != nil { - globalNode := &api.Node{ - Links: globalLinksConfig, - } - pruneModuleLinks(moduleLinks.Rewrites, node, getNodeRewrites) - pruneModuleLinks(moduleLinks.Rewrites, globalNode, getNodeRewrites) - if moduleLinks.Downloads != nil { - pruneModuleLinks(moduleLinks.Downloads.Renames, node, getNodeDownloadsRenamesKeys) - pruneModuleLinks(moduleLinks.Downloads.Renames, globalNode, getNodeDownloadsRenamesKeys) - pruneModuleLinks(moduleLinks.Downloads.Scope, node, getNodeDownloadsScopeKeys) - pruneModuleLinks(moduleLinks.Downloads.Scope, globalNode, getNodeDownloadsScopeKeys) - } - } - newNode.Links = moduleDocumentation.Links - if moduleDocumentation.NodeSelector != nil { - childNode := &api.Node{ - NodeSelector: moduleDocumentation.NodeSelector, - } - childNode.SetParent(node) - res, err := r.resolveNodeSelector(ctx, childNode, globalLinksConfig) - if err != nil { - return nil, err - } - for _, n := range res.Nodes { - n.SetParent(node) - n.SetParentsDownwards() - } - - pruneChildNodesLinks(node, res.Nodes, globalLinksConfig) - newNode.Nodes = append(newNode.Nodes, res.Nodes...) - } - return newNode, nil - } - - nodes, err := handler.ResolveNodeSelector(ctx, node, node.NodeSelector.ExcludePaths, node.NodeSelector.ExcludeFrontMatter, node.NodeSelector.FrontMatter, node.NodeSelector.Depth) - if err != nil { - return nil, err - } - - newNode.Nodes = nodes - return newNode, nil -} - -func (r *Reactor) resolveNodeName(ctx context.Context, node *api.Node) (string, error) { - name := node.Name - handler := r.ResourceHandlers.Get(node.Source) - if handler == nil { - return "", fmt.Errorf("No suitable handler registered for URL %s", node.Source) - } - if len(node.Name) == 0 { - name = "$name" - } - resourceName, ext := handler.ResourceName(node.Source) - id := uuid.New().String() - name = strings.ReplaceAll(name, "$name", resourceName) - name = strings.ReplaceAll(name, "$uuid", id) - name = strings.ReplaceAll(name, "$ext", fmt.Sprintf(".%s", ext)) - return name, nil -} - -func pruneModuleLinks(moduleLinks interface{}, node *api.Node, getParentLinks func(node *api.Node) map[string]struct{}) { - v := reflect.ValueOf(moduleLinks) - if v.Kind() != reflect.Map { - return - } - - for _, moduleLinkKey := range v.MapKeys() { - for parent := node; parent != nil; parent = parent.Parent() { - if parent.Links == nil { - continue - } - - if parentLinks := getParentLinks(parent); parentLinks != nil { - if _, ok := parentLinks[moduleLinkKey.String()]; ok { - klog.Warningf("Overriding module link %s", moduleLinkKey.String()) - v.SetMapIndex(moduleLinkKey, reflect.Value{}) - } - } - } - } -} - -func getNodeRewrites(node *api.Node) map[string]struct{} { - if node.Links == nil { - return nil - } - keys := make(map[string]struct{}) - for k := range node.Links.Rewrites { - keys[k] = struct{}{} - } - return keys -} - -func getNodeDownloadsRenamesKeys(node *api.Node) map[string]struct{} { - if node.Links == nil { - return nil - } - if node.Links.Downloads == nil { - return nil - } - - keys := make(map[string]struct{}) - for k := range node.Links.Downloads.Renames { - keys[k] = struct{}{} - } - return keys -} - -func getNodeDownloadsScopeKeys(node *api.Node) map[string]struct{} { - if node.Links == nil { - return nil - } - if node.Links.Downloads == nil { - return nil - } - - keys := make(map[string]struct{}) - for k := range node.Links.Downloads.Scope { - keys[k] = struct{}{} - } - return keys -} - -func pruneChildNodesLinks(parentNode *api.Node, nodes []*api.Node, globalLinksConfig *api.Links) { - for _, node := range nodes { - if node.Nodes != nil { - pruneChildNodesLinks(node, node.Nodes, globalLinksConfig) - } - - if node.Links != nil { - globalNode := &api.Node{ - Links: globalLinksConfig, - } - pruneModuleLinks(node.Links.Rewrites, parentNode, getNodeRewrites) - pruneModuleLinks(node.Links.Rewrites, globalNode, getNodeRewrites) - if node.Links.Downloads != nil { - pruneModuleLinks(node.Links.Downloads.Renames, parentNode, getNodeDownloadsRenamesKeys) - pruneModuleLinks(node.Links.Downloads.Renames, globalNode, getNodeDownloadsRenamesKeys) - pruneModuleLinks(node.Links.Downloads.Scope, parentNode, getNodeDownloadsScopeKeys) - pruneModuleLinks(node.Links.Downloads.Scope, globalNode, getNodeDownloadsScopeKeys) - } - } - } -} diff --git a/pkg/resourcehandlers/github/github_resource_handler.go b/pkg/resourcehandlers/github/github_resource_handler.go index 1efb8c28..8e54a6da 100644 --- a/pkg/resourcehandlers/github/github_resource_handler.go +++ b/pkg/resourcehandlers/github/github_resource_handler.go @@ -352,11 +352,11 @@ func (gh *GitHub) Accept(uri string) bool { func (gh *GitHub) ResolveNodeSelector(ctx context.Context, node *api.Node, excludePaths []string, frontMatter map[string]interface{}, excludeFrontMatter map[string]interface{}, depth int32) ([]*api.Node, error) { rl, err := gh.URLToGitHubLocator(ctx, node.NodeSelector.Path, true) if err != nil { + if err == resourcehandlers.ErrResourceNotFound { + return []*api.Node{}, nil + } return nil, err } - if rl == nil { - return nil, nil - } childResourceLocators := gh.cache.GetSubset(rl.String()) childNodes, err := buildNodes(node, excludePaths, frontMatter, excludeFrontMatter, depth, childResourceLocators, gh.cache, 0) @@ -368,7 +368,7 @@ func (gh *GitHub) ResolveNodeSelector(ctx context.Context, node *api.Node, exclu cleanupNodeTree(child) } if childNodes == nil { - return nil, nil + return []*api.Node{}, nil } return childNodes, nil diff --git a/pkg/resourcehandlers/resource_handlers.go b/pkg/resourcehandlers/resource_handlers.go index 59cb7cd3..a74564ca 100644 --- a/pkg/resourcehandlers/resource_handlers.go +++ b/pkg/resourcehandlers/resource_handlers.go @@ -12,7 +12,7 @@ import ( "github.com/gardener/docforge/pkg/api" ) -// ErrIsResourceNotFound indicated that a resource was not found +// ErrResourceNotFound indicated that a resource was not found var ErrResourceNotFound = errors.New("resource not found") // ResourceHandler does resource specific operations on a type of objects diff --git a/pkg/resourcehandlers/testhandler/test_resource_handler.go b/pkg/resourcehandlers/testhandler/test_resource_handler.go new file mode 100644 index 00000000..b10b661b --- /dev/null +++ b/pkg/resourcehandlers/testhandler/test_resource_handler.go @@ -0,0 +1,96 @@ +package testhandler + +import ( + "context" + + "github.com/gardener/docforge/pkg/api" +) + +//TestResourceHandler ... +type TestResourceHandler struct { + accept func(uri string) bool + resolveNodeSelector func(ctx context.Context, node *api.Node, excludePaths []string, frontMatter map[string]interface{}, excludeFrontMatter map[string]interface{}, depth int32) ([]*api.Node, error) + resolveDocumentation func(ctx context.Context, uri string) (*api.Documentation, error) +} + +//NewTestResouceHandlere ... +func NewTestResouceHandlere() *TestResourceHandler { + return &TestResourceHandler{} +} + +//Accept ... +func (t *TestResourceHandler) Accept(uri string) bool { + if t.accept != nil { + return t.accept(uri) + } + return true +} + +//WithAccept ... +func (t *TestResourceHandler) WithAccept(accept func(uri string) bool) *TestResourceHandler { + t.accept = accept + return t +} + +//WithResolveNodeSelector ... +func (t *TestResourceHandler) WithResolveNodeSelector(resolveNodeSelector func(ctx context.Context, node *api.Node, excludePaths []string, frontMatter map[string]interface{}, excludeFrontMatter map[string]interface{}, depth int32) ([]*api.Node, error)) *TestResourceHandler { + t.resolveNodeSelector = resolveNodeSelector + return t +} + +//ResolveNodeSelector ... +func (t *TestResourceHandler) ResolveNodeSelector(ctx context.Context, node *api.Node, excludePaths []string, frontMatter map[string]interface{}, excludeFrontMatter map[string]interface{}, depth int32) ([]*api.Node, error) { + if t.resolveNodeSelector != nil { + return t.resolveNodeSelector(ctx, node, excludePaths, frontMatter, excludeFrontMatter, depth) + } + return []*api.Node{}, nil +} + +// Read a resource content at uri into a byte array +func (t *TestResourceHandler) Read(ctx context.Context, uri string) ([]byte, error) { + return []byte{}, nil +} + +// ReadGitInfo .. +func (t *TestResourceHandler) ReadGitInfo(ctx context.Context, uri string) ([]byte, error) { + return []byte{}, nil +} + +// ResourceName returns a breakdown of a resource name in the link, consisting +// of name and potentially and extention without the dot. +func (t *TestResourceHandler) ResourceName(link string) (string, string) { + return "", "" +} + +// BuildAbsLink should return an absolute path of a relative link in regards of the provided +// source +func (t *TestResourceHandler) BuildAbsLink(source, link string) (string, error) { + return "", nil +} + +// GetRawFormatLink returns a link to an embedable object (image) in raw format. +// If the provided link is not referencing an embedable object, the function +// returns absLink without changes. +func (t *TestResourceHandler) GetRawFormatLink(absLink string) (string, error) { + return "", nil +} + +// SetVersion sets version to absLink according to the API scheme. For GitHub +// for example this would replace e.g. the 'master' segment in the path with version +func (t *TestResourceHandler) SetVersion(absLink, version string) (string, error) { + return "", nil +} + +// ResolveDocumentation for a given uri +func (t *TestResourceHandler) ResolveDocumentation(ctx context.Context, uri string) (*api.Documentation, error) { + if t.resolveDocumentation != nil { + return t.resolveDocumentation(ctx, uri) + } + return nil, nil +} + +//WithResolveDocumentation overriding the default TestResourceHandler function +func (t *TestResourceHandler) WithResolveDocumentation(resolveDocumentation func(ctx context.Context, uri string) (*api.Documentation, error)) *TestResourceHandler { + t.resolveDocumentation = resolveDocumentation + return t +}