Change --dryRun semantics and add dedicated --resolve

cmd/app/cmd.go

@@ -3,6 +3,8 @@ package app
 import (
 	"context"
 	"flag"
+	"io"
+	"os"
 
 	"github.com/gardener/docforge/pkg/hugo"
 	"github.com/spf13/cobra"
@@ -20,6 +22,7 @@ type cmdFlags struct {
 	markdownFmt                  bool
 	ghOAuthToken                 string
 	dryRun                       bool
+	resolve                      bool
 	clientMetering               bool
 	hugo                         bool
 	hugoPrettyUrls               bool
@@ -71,7 +74,9 @@ func (flags *cmdFlags) Configure(command *cobra.Command) {
 	command.Flags().BoolVar(&flags.markdownFmt, "markdownfmt", true,
 		"Applies formatting rules to source markdown.")
 	command.Flags().BoolVar(&flags.dryRun, "dry-run", false,
-		"Resolves and prints the resolved documentation structure without downloading anything.")
+		"Runs the command end-to-end but instead of writing files, it will output the proejcted file/folder hierarchy to the standard output and statistics for the processing of each file.")
+	command.Flags().BoolVar(&flags.resolve, "resolve", false,
+		"Resolves the documentation structure and prints it to the standard output. The resolution expands nodeSelector constructs into node hierarchies.")
 	command.Flags().IntVar(&flags.minWorkersCount, "min-workers", 10,
 		"Minimum number of parallel workers.")
 	command.Flags().IntVar(&flags.maxWorkersCount, "max-workers", 25,
@@ -92,9 +97,10 @@ func (flags *cmdFlags) Configure(command *cobra.Command) {
 // NewOptions creates an options object from flags
 func NewOptions(f *cmdFlags) *Options {
 	var (
-		tokens      map[string]string
-		metering    *Metering
-		hugoOptions *hugo.Options
+		tokens       map[string]string
+		metering     *Metering
+		hugoOptions  *hugo.Options
+		dryRunWriter io.Writer
 	)
 	if len(f.ghOAuthToken) > 0 {
 		tokens = map[string]string{
@@ -115,6 +121,10 @@ func NewOptions(f *cmdFlags) *Options {
 		}
 	}
 
+	if f.dryRun {
+		dryRunWriter = os.Stdout
+	}
+
 	return &Options{
 		DestinationPath:              f.destinationPath,
 		FailFast:                     f.failFast,
@@ -125,6 +135,8 @@ func NewOptions(f *cmdFlags) *Options {
 		MarkdownFmt:                  f.markdownFmt,
 		GitHubTokens:                 tokens,
 		Metering:                     metering,
+		DryRunWriter:                 dryRunWriter,
+		Resolve:                      f.resolve,
 		Hugo:                         hugoOptions,
 	}
 }

cmd/app/factory.go

@@ -2,6 +2,7 @@ package app
 
 import (
 	"context"
+	"io"
 	"path/filepath"
 
 	"github.com/gardener/docforge/pkg/hugo"
@@ -30,6 +31,8 @@ type Options struct {
 	MarkdownFmt                  bool
 	GitHubTokens                 map[string]string
 	Metering                     *Metering
+	DryRunWriter                 io.Writer
+	Resolve                      bool
 	Hugo                         *hugo.Options
 }
 
@@ -41,6 +44,7 @@ type Metering struct {
 
 // NewReactor creates a Reactor from Options
 func NewReactor(ctx context.Context, options *Options) *reactor.Reactor {
+	dryRunWriters := writers.NewDryRunWritersFactory(options.DryRunWriter)
 	o := &reactor.Options{
 		MaxWorkersCount:              options.MaxWorkersCount,
 		MinWorkersCount:              options.MinWorkersCount,
@@ -50,13 +54,20 @@ func NewReactor(ctx context.Context, options *Options) *reactor.Reactor {
 		ResourceDownloadWorkersCount: options.ResourceDownloadWorkersCount,
 		MarkdownFmt:                  options.MarkdownFmt,
 		Processor:                    nil,
-		Writer: &writers.FSWriter{
+		ResourceHandlers:             initResourceHanlders(ctx, options),
+		DryRunWriter:                 dryRunWriters,
+		Resolve:                      options.Resolve,
+	}
+	if options.DryRunWriter != nil {
+		o.Writer = dryRunWriters.GetWriter(options.DestinationPath)
+		o.ResourceDownloadWriter = dryRunWriters.GetWriter(filepath.Join(options.DestinationPath, options.ResourcesPath))
+	} else {
+		o.Writer = &writers.FSWriter{
 			Root: options.DestinationPath,
-		},
-		ResourceDownloadWriter: &writers.FSWriter{
+		}
+		o.ResourceDownloadWriter = &writers.FSWriter{
 			Root: filepath.Join(options.DestinationPath, options.ResourcesPath),
-		},
-		ResourceHandlers: initResourceHanlders(ctx, options),
+		}
 	}
 
 	if options.Hugo != nil {
@@ -76,8 +87,12 @@ func WithHugo(reactorOptions *reactor.Options, o *Options) {
 			hugo.NewProcessor(hugoOptions),
 		},
 	}
-	hugoOptions.Writer = &writers.FSWriter{
-		Root: filepath.Join(o.DestinationPath),
+	if o.DryRunWriter != nil {
+		hugoOptions.Writer = reactorOptions.Writer
+	} else {
+		hugoOptions.Writer = &writers.FSWriter{
+			Root: filepath.Join(o.DestinationPath),
+		}
 	}
 	reactorOptions.Writer = hugo.NewWriter(hugoOptions)
 }

pkg/api/nodes.go

@@ -130,6 +130,18 @@ func (n *Node) Peers() []*Node {
 	return peers
 }
 
+// GetStats returns statistics for this node
+func (n *Node) GetStats() []*Stat {
+	return n.stats
+}
+
+// AddStats appends Stats
+func (n *Node) AddStats(s ...*Stat) {
+	for _, stat := range s {
+		n.stats = append(n.stats, stat)
+	}
+}
+
 // FindNodeByContentSource traverses up and then all around the
 // tree paths in the node's documentation strcuture, looking for
 // a node that has contentSource path nodeContentSource

pkg/api/stats.go

@@ -0,0 +1,8 @@
+package api
+
+// Stat represents a category recorded by StatsRecorder
+type Stat struct {
+	Title   string
+	Figures string
+	Details []string
+}

pkg/api/types.go

@@ -105,6 +105,8 @@ type Node struct {
 	// their absolute form for the match
 	// TODO: update this doc
 	LinksSubstitutes LinkSubstitutes `yaml:"linksSubstitutes,omitempty"`
+
+	stats []*Stat
 }
 
 // NodeSelector is an specification for selecting subnodes (children) for a node.
@@ -195,6 +197,8 @@ type LocalityDomain struct {
 	DownloadSubstitutes map[string]string `yaml:"downloadSubstitutes,omitempty"`
 }
 
+// LocalityDomainMap maps domains such as github.com/gardener/gardener
+// to LocalityDomainValues
 type LocalityDomainMap map[string]*LocalityDomainValue
 
 // LocalityDomainValue encapsulates the memebers of a
@@ -211,8 +215,12 @@ type LocalityDomainValue struct {
 	LinksMatchers `yaml:",inline"`
 }
 
+// LinkSubstitutes is the mapping between absolute links
+// and substitutions for them
 type LinkSubstitutes map[string]*LinkSubstitute
 
+// LinkSubstitute comprises subtitutes for various link details
+// commonly found in markup
 type LinkSubstitute struct {
 	Text        *string `yaml:"text,omitempty"`
 	Destination *string `yaml:"destination,omitempty"`

pkg/jobs/controller_test.go

@@ -82,13 +82,13 @@ func TestController(t *testing.T) {
 			ctx, cancel := context.WithTimeout(context.Background(), tc.timeout)
 			defer cancel()
 			job := &Job{
-				FailFast:          tc.failFast,
-				ID:                "Test",
-				MaxWorkers:        tc.workersCount,
-				MinWorkers:        tc.workersCount,
+				FailFast:                  tc.failFast,
+				ID:                        "Test",
+				MaxWorkers:                tc.workersCount,
+				MinWorkers:                tc.workersCount,
 				IsWorkerExitsOnEmptyQueue: true,
-				Worker:            WorkerFunc(tc.worker.work),
-				Queue:             NewWorkQueue(tc.tasksCount),
+				Worker:                    WorkerFunc(tc.worker.work),
+				Queue:                     NewWorkQueue(tc.tasksCount),
 			}
 
 			c := NewController(job)

pkg/reactor/content_processor.go

@@ -109,6 +109,13 @@ func (c *NodeContentProcessor) reconcileMDLinks(ctx context.Context, docNode *ap
 				return destination, text, title, err
 			}
 		}
+		if docNode != nil {
+			if _destination != string(destination) {
+				recordLinkStats(docNode, "Links", fmt.Sprintf("%s -> %s", string(destination), _destination))
+			} else {
+				recordLinkStats(docNode, "Links", "")
+			}
+		}
 		if download != nil {
 			c.schedule(ctx, download, contentSourcePath)
 		}
@@ -144,6 +151,13 @@ func (c *NodeContentProcessor) reconcileHTMLLinks(ctx context.Context, docNode *
 			}
 			destination, _, _, download, err := c.resolveLink(ctx, docNode, url, contentSourcePath)
 			klog.V(6).Infof("[%s] %s -> %s\n", contentSourcePath, url, destination)
+			if docNode != nil {
+				if url != destination {
+					recordLinkStats(docNode, "Links", fmt.Sprintf("%s -> %s", url, destination))
+				} else {
+					recordLinkStats(docNode, "Links", "")
+				}
+			}
 			if download != nil {
 				c.schedule(ctx, download, contentSourcePath)
 			}
@@ -157,6 +171,7 @@ func (c *NodeContentProcessor) reconcileHTMLLinks(ctx context.Context, docNode *
 	return documentBytes, errors.ErrorOrNil()
 }
 
+// Download represents a resource that can be downloaded
 type Download struct {
 	url          string
 	resourceName string
@@ -167,6 +182,8 @@ func (c *NodeContentProcessor) resolveLink(ctx context.Context, node *api.Node,
 	var (
 		text, title, substituteDestination *string
 		hasSubstition                      bool
+		inLD                               bool
+		absLink                            string
 	)
 	if strings.HasPrefix(destination, "#") || strings.HasPrefix(destination, "mailto:") {
 		return destination, nil, nil, nil, nil
@@ -196,11 +213,13 @@ func (c *NodeContentProcessor) resolveLink(ctx context.Context, node *api.Node,
 	}
 	_a := absLink
 
-	recolvedLD := c.localityDomain
+	resolvedLD := c.localityDomain
 	if node != nil {
-		recolvedLD = resolveLocalityDomain(node, c.localityDomain)
+		resolvedLD = resolveLocalityDomain(node, c.localityDomain)
+	}
+	if resolvedLD != nil {
+		absLink, inLD = resolvedLD.MatchPathInLocality(absLink, c.ResourceHandlers)
 	}
-	absLink, inLD := recolvedLD.MatchPathInLocality(absLink, c.ResourceHandlers)
 	if _a != absLink {
 		klog.V(6).Infof("[%s] Link converted %s -> %s\n", contentSourcePath, _a, absLink)
 	}
@@ -227,7 +246,7 @@ func (c *NodeContentProcessor) resolveLink(ctx context.Context, node *api.Node,
 	// and if applicable their destination is updated as relative
 	// path to predefined location for resources
 	if absLink != "" && inLD {
-		resourceName := c.generateResourceName(absLink, recolvedLD)
+		resourceName := c.generateResourceName(absLink, resolvedLD)
 		_d := destination
 		destination = buildDestination(node, resourceName, c.resourcesRoot)
 		if _d != destination {
@@ -296,3 +315,36 @@ func substitute(absLink string, node *api.Node) (ok bool, destination *string, t
 	}
 	return false, nil, nil, nil
 }
+
+// recordLinkStats records link stats for a node
+func recordLinkStats(node *api.Node, title, details string) {
+	var (
+		stat *api.Stat
+	)
+	nodeStats := node.GetStats()
+	if nodeStats != nil {
+		for _, _stat := range nodeStats {
+			if _stat.Title == title {
+				stat = _stat
+				break
+			}
+		}
+	}
+	if stat == nil {
+		stat = &api.Stat{
+			Title: title,
+		}
+		if len(details) > 0 {
+			stat.Details = []string{details}
+		} else {
+			stat.Details = []string{}
+		}
+		stat.Figures = fmt.Sprintf("%d link rewrites", len(stat.Details))
+		node.AddStats(stat)
+		return
+	}
+	if len(details) > 0 {
+		stat.Details = append(stat.Details, details)
+	}
+	stat.Figures = fmt.Sprintf("%d link rewrites", len(stat.Details))
+}