diff --git a/cmd/app/cmd.go b/cmd/app/cmd.go index f07c9d32..fd47e53f 100644 --- a/cmd/app/cmd.go +++ b/cmd/app/cmd.go @@ -22,6 +22,7 @@ type cmdFlags struct { markdownFmt bool ghOAuthToken string dryRun bool + resolve bool clientMetering bool hugo bool hugoPrettyUrls bool @@ -74,6 +75,8 @@ func (flags *cmdFlags) Configure(command *cobra.Command) { "Applies formatting rules to source markdown.") command.Flags().BoolVar(&flags.dryRun, "dry-run", false, "Runs the command end-to-end but instead of writing files, it will output the proejcted file/folder hierarchy to the standard output and statistics for the processing of each file.") + command.Flags().BoolVar(&flags.resolve, "resolve", false, + "Resolves the documentation structure and prints it to the standard output. The resolution expands nodeSelector constructs into node hierarchies.") command.Flags().IntVar(&flags.minWorkersCount, "min-workers", 10, "Minimum number of parallel workers.") command.Flags().IntVar(&flags.maxWorkersCount, "max-workers", 25, @@ -133,6 +136,7 @@ func NewOptions(f *cmdFlags) *Options { GitHubTokens: tokens, Metering: metering, DryRunWriter: dryRunWriter, + Resolve: f.resolve, Hugo: hugoOptions, } } diff --git a/cmd/app/factory.go b/cmd/app/factory.go index df3fe317..1b441cf4 100644 --- a/cmd/app/factory.go +++ b/cmd/app/factory.go @@ -32,6 +32,7 @@ type Options struct { GitHubTokens map[string]string Metering *Metering DryRunWriter io.Writer + Resolve bool Hugo *hugo.Options } @@ -55,6 +56,7 @@ func NewReactor(ctx context.Context, options *Options) *reactor.Reactor { Processor: nil, ResourceHandlers: initResourceHanlders(ctx, options), DryRunWriter: dryRunWriters, + Resolve: options.Resolve, } if options.DryRunWriter != nil { o.Writer = dryRunWriters.GetWriter(options.DestinationPath) diff --git a/pkg/reactor/content_processor.go b/pkg/reactor/content_processor.go index 84ff593c..3d2661c3 100644 --- a/pkg/reactor/content_processor.go +++ b/pkg/reactor/content_processor.go @@ -111,7 +111,7 @@ func (c *NodeContentProcessor) reconcileMDLinks(ctx context.Context, docNode *ap } if docNode != nil { if _destination != string(destination) { - recordLinkStats(docNode, "Links", fmt.Sprintf("%s -> %s", _destination, string(destination))) + recordLinkStats(docNode, "Links", fmt.Sprintf("%s -> %s", string(destination), _destination)) } else { recordLinkStats(docNode, "Links", "") } @@ -151,6 +151,13 @@ func (c *NodeContentProcessor) reconcileHTMLLinks(ctx context.Context, docNode * } destination, _, _, download, err := c.resolveLink(ctx, docNode, url, contentSourcePath) klog.V(6).Infof("[%s] %s -> %s\n", contentSourcePath, url, destination) + if docNode != nil { + if url != destination { + recordLinkStats(docNode, "Links", fmt.Sprintf("%s -> %s", url, destination)) + } else { + recordLinkStats(docNode, "Links", "") + } + } if download != nil { c.schedule(ctx, download, contentSourcePath) } @@ -206,12 +213,12 @@ func (c *NodeContentProcessor) resolveLink(ctx context.Context, node *api.Node, } _a := absLink - recolvedLD := c.localityDomain + resolvedLD := c.localityDomain if node != nil { - recolvedLD = resolveLocalityDomain(node, c.localityDomain) + resolvedLD = resolveLocalityDomain(node, c.localityDomain) } - if recolvedLD != nil { - absLink, inLD = recolvedLD.MatchPathInLocality(absLink, c.ResourceHandlers) + if resolvedLD != nil { + absLink, inLD = resolvedLD.MatchPathInLocality(absLink, c.ResourceHandlers) } if _a != absLink { klog.V(6).Infof("[%s] Link converted %s -> %s\n", contentSourcePath, _a, absLink) @@ -239,7 +246,7 @@ func (c *NodeContentProcessor) resolveLink(ctx context.Context, node *api.Node, // and if applicable their destination is updated as relative // path to predefined location for resources if absLink != "" && inLD { - resourceName := c.generateResourceName(absLink, recolvedLD) + resourceName := c.generateResourceName(absLink, resolvedLD) _d := destination destination = buildDestination(node, resourceName, c.resourcesRoot) if _d != destination { diff --git a/pkg/reactor/reactor.go b/pkg/reactor/reactor.go index c79fbd6a..c97d8bee 100644 --- a/pkg/reactor/reactor.go +++ b/pkg/reactor/reactor.go @@ -28,6 +28,7 @@ type Options struct { Writer writers.Writer ResourceHandlers []resourcehandlers.ResourceHandler DryRunWriter writers.DryRunWriter + Resolve bool } // NewReactor creates a Reactor from Options @@ -47,6 +48,7 @@ func NewReactor(o *Options) *Reactor { DocController: docController, DownloadController: downloadController, DryRunWriter: o.DryRunWriter, + Resolve: o.Resolve, } return r } @@ -59,6 +61,7 @@ type Reactor struct { DocController DocumentController DownloadController DownloadController DryRunWriter writers.DryRunWriter + Resolve bool } // Run starts build operation on docStruct @@ -67,7 +70,7 @@ func (r *Reactor) Run(ctx context.Context, docStruct *api.Documentation, dryRun err error ld *localityDomain ) - if err := r.Resolve(ctx, docStruct.Root); err != nil { + if err := r.ResolveStructure(ctx, docStruct.Root); err != nil { return err } @@ -81,12 +84,11 @@ func (r *Reactor) Run(ctx context.Context, docStruct *api.Documentation, dryRun } } - if dryRun { + if r.Resolve { s, err := api.Serialize(docStruct) if err != nil { return err } - // TODO: either reuse dry run's wrapped writer or build different command for that os.Stdout.Write([]byte(s)) os.Stdout.Write([]byte("\n\n")) } @@ -106,12 +108,12 @@ func (r *Reactor) Run(ctx context.Context, docStruct *api.Documentation, dryRun return nil } -// Resolve builds the subnodes hierarchy of a node based on the natural nodes +// ResolveStructure builds the subnodes hierarchy of a node based on the natural nodes // hierarchy and on rules such as those in NodeSelector. // The node hierarchy is resolved by an appropriate handler selected based // on the NodeSelector path URI // The resulting model is the actual flight plan for replicating resources. -func (r *Reactor) Resolve(ctx context.Context, node *api.Node) error { +func (r *Reactor) ResolveStructure(ctx context.Context, node *api.Node) error { node.SetParentsDownwards() if node.NodeSelector != nil { var handler resourcehandlers.ResourceHandler @@ -121,10 +123,12 @@ func (r *Reactor) Resolve(ctx context.Context, node *api.Node) error { if err := handler.ResolveNodeSelector(ctx, node); err != nil { return err } + // remove node selctors after resolution + node.NodeSelector = nil } if len(node.Nodes) > 0 { for _, n := range node.Nodes { - if err := r.Resolve(ctx, n); err != nil { + if err := r.ResolveStructure(ctx, n); err != nil { return err } } diff --git a/pkg/writers/dryRunWriter.go b/pkg/writers/dryRunWriter.go index 6e37022b..265bc58b 100644 --- a/pkg/writers/dryRunWriter.go +++ b/pkg/writers/dryRunWriter.go @@ -5,6 +5,7 @@ import ( "fmt" "io" "io/ioutil" + "path/filepath" "sort" "strings" "time" @@ -67,12 +68,20 @@ func (d *dryRunWriter) GetWriter(root string) Writer { } func (w *writer) Write(name, path string, docBlob []byte, node *api.Node) error { - if len(docBlob) > 0 && node != nil && !strings.HasSuffix(name, ".md") { - name = fmt.Sprintf("%s.md", name) + var stats []*api.Stat + if len(docBlob) > 0 && node != nil { + if !strings.HasSuffix(name, ".md") { + name = fmt.Sprintf("%s.md", name) + } + stats = node.GetStats() } + root := filepath.Clean(w.root) + path = filepath.Clean(path) + filePath := fmt.Sprintf("%s/%s/%s", root, path, name) + filePath = filepath.Clean(filePath) f := &file{ - path: fmt.Sprintf("%s/%s/%s", w.root, path, name), - stats: node.GetStats(), + path: filePath, + stats: stats, } *w.files = append(*w.files, f) return nil @@ -107,6 +116,7 @@ func format(files []*file, b *bytes.Buffer) { all := []string{} for _, f := range files { p := f.path + p = filepath.Clean(p) dd := strings.Split(p, "/") indent := 0 for i, s := range dd { @@ -119,6 +129,9 @@ func format(files []*file, b *bytes.Buffer) { if !any(all, _p) { all = append(all, _p) b.WriteString(fmt.Sprintf("%s\n", s)) + if i < len(dd)-1 { + b.Write(bytes.Repeat([]byte(" "), i)) + } for _, st := range f.stats { b.Write([]byte(" ")) b.Write(bytes.Repeat([]byte(" "), indent)) diff --git a/pkg/writers/dryRunWriter_test.go b/pkg/writers/dryRunWriter_test.go index f785e5ea..ae16a5a4 100644 --- a/pkg/writers/dryRunWriter_test.go +++ b/pkg/writers/dryRunWriter_test.go @@ -2,6 +2,7 @@ package writers import ( "bytes" + "fmt" "io/ioutil" "testing" @@ -15,7 +16,8 @@ func TestFormat(t *testing.T) { err error ) in := []string{ - "dev/doc", + "dev/__resources/015ec383-3c1b-487b-acff-4d7f4f8a1b14.png", + "dev/__resources/173a7246-e1d5-40d5-b981-8cff293e177a.png", "dev/doc/aws_provider.md", "dev/doc/gardener", "dev/doc/gardener/_index.md", @@ -36,6 +38,9 @@ func TestFormat(t *testing.T) { "dev/doc/gardener/usage/control_plane_migration.md", } out := `dev + __resources + 015ec383-3c1b-487b-acff-4d7f4f8a1b14.png + 173a7246-e1d5-40d5-b981-8cff293e177a.png doc aws_provider.md gardener @@ -63,10 +68,12 @@ func TestFormat(t *testing.T) { path: p, }) } + format(files, &b) if bytes, err = ioutil.ReadAll(&b); err != nil { t.Error(err.Error()) } assert.Equal(t, out, string(bytes)) + fmt.Printf("%s\n", string(bytes)) }