support ?raw=true and all GH object types

cmd/app/cmd.go

@@ -20,6 +20,7 @@ type cmdFlags struct {
 	resourcesPath                string
 	resourceDownloadWorkersCount int
 	markdownFmt                  bool
+	rewriteEmbedded              bool
 	ghOAuthToken                 string
 	dryRun                       bool
 	resolve                      bool
@@ -69,6 +70,8 @@ func (flags *cmdFlags) Configure(command *cobra.Command) {
 		"Resources download path.")
 	command.Flags().StringVar(&flags.ghOAuthToken, "github-oauth-token", "",
 		"GitHub personal token authorizing reading from GitHub repositories.")
+	command.Flags().BoolVar(&flags.rewriteEmbedded, "rewrite-embedded-to-raw", true,
+		"Rewrites absolute link destinations for embedded resources (images) to reference embeddable media (e.g. for GitHub - reference to a 'raw' version of an image).")
 	command.Flags().BoolVar(&flags.failFast, "fail-fast", false,
 		"Fail-fast vs fault tolerant operation.")
 	command.Flags().BoolVar(&flags.markdownFmt, "markdownfmt", true,
@@ -133,6 +136,7 @@ func NewOptions(f *cmdFlags) *Options {
 		ResourceDownloadWorkersCount: f.resourceDownloadWorkersCount,
 		ResourcesPath:                f.resourcesPath,
 		MarkdownFmt:                  f.markdownFmt,
+		RewriteEmbedded:              f.rewriteEmbedded,
 		GitHubTokens:                 tokens,
 		Metering:                     metering,
 		DryRunWriter:                 dryRunWriter,

cmd/app/factory.go

@@ -29,6 +29,7 @@ type Options struct {
 	ResourcesPath                string
 	ResourceDownloadWorkersCount int
 	MarkdownFmt                  bool
+	RewriteEmbedded              bool
 	GitHubTokens                 map[string]string
 	Metering                     *Metering
 	DryRunWriter                 io.Writer
@@ -53,6 +54,7 @@ func NewReactor(ctx context.Context, options *Options) *reactor.Reactor {
 		ResourcesPath:                options.ResourcesPath,
 		ResourceDownloadWorkersCount: options.ResourceDownloadWorkersCount,
 		MarkdownFmt:                  options.MarkdownFmt,
+		RewriteEmbedded:              options.RewriteEmbedded,
 		Processor:                    nil,
 		ResourceHandlers:             initResourceHandlers(ctx, options),
 		DryRunWriter:                 dryRunWriters,

pkg/reactor/content_processor.go

@@ -37,11 +37,12 @@ type NodeContentProcessor struct {
 	DownloadController DownloadController
 	failFast           bool
 	markdownFmt        bool
+	rewriteEmbedded    bool
 	ResourceHandlers   resourcehandlers.Registry
 }
 
 // NewNodeContentProcessor creates NodeContentProcessor objects
-func NewNodeContentProcessor(resourcesRoot string, ld *localityDomain, downloadJob DownloadController, failFast bool, markdownFmt bool, resourceHandlers resourcehandlers.Registry) *NodeContentProcessor {
+func NewNodeContentProcessor(resourcesRoot string, ld *localityDomain, downloadJob DownloadController, failFast bool, markdownFmt bool, rewriteEmbedded bool, resourceHandlers resourcehandlers.Registry) *NodeContentProcessor {
 	if ld == nil {
 		ld = &localityDomain{
 			mapping: map[string]*localityDomainValue{},
@@ -54,6 +55,7 @@ func NewNodeContentProcessor(resourcesRoot string, ld *localityDomain, downloadJ
 		DownloadController: downloadJob,
 		failFast:           failFast,
 		markdownFmt:        markdownFmt,
+		rewriteEmbedded:    rewriteEmbedded,
 		ResourceHandlers:   resourceHandlers,
 	}
 	return c
@@ -189,13 +191,24 @@ func (c *NodeContentProcessor) resolveLink(ctx context.Context, node *api.Node,
 		return destination, nil, nil, nil, nil
 	}
 
+	// validate destination
+	u, err := url.Parse(destination)
+	if err != nil {
+		return "", text, title, nil, err
+	}
+	// can we handle this destination?
+	if u.IsAbs() && c.ResourceHandlers.Get(destination) == nil {
+		// It's a valid absolute link that is not in our scope. Leave it be.
+		return destination, text, title, nil, err
+	}
+
 	handler := c.ResourceHandlers.Get(contentSourcePath)
 	if handler == nil {
-		return destination, nil, nil, nil, nil
+		return destination, text, title, nil, nil
 	}
-	absLink, err := handler.BuildAbsLink(contentSourcePath, destination)
+	absLink, err = handler.BuildAbsLink(contentSourcePath, destination)
 	if err != nil {
-		return "", nil, nil, nil, err
+		return "", text, title, nil, err
 	}
 
 	if hasSubstition, substituteDestination, text, title = substitute(absLink, node); hasSubstition && substituteDestination != nil {
@@ -207,7 +220,7 @@ func (c *NodeContentProcessor) resolveLink(ctx context.Context, node *api.Node,
 	}
 
 	//TODO: this is URI-specific (URLs only) - fixme
-	u, err := url.Parse(absLink)
+	u, err = url.Parse(absLink)
 	if err != nil {
 		return "", text, title, nil, err
 	}
@@ -254,6 +267,13 @@ func (c *NodeContentProcessor) resolveLink(ctx context.Context, node *api.Node,
 		}
 		return destination, text, title, &Download{absLink, resourceName}, nil
 	}
+
+	// rewrite abs links to embedded objects to their raw format if necessary, to
+	// ensure they are embedable
+	if absLink, err = handler.GetRawFormatLink(absLink); err != nil {
+		return "", text, title, nil, err
+	}
+
 	if destination != absLink {
 		klog.V(6).Infof("[%s] %s -> %s\n", contentSourcePath, destination, absLink)
 	}

pkg/reactor/content_processor_test.go

@@ -84,7 +84,7 @@ func Test_processLink(t *testing.T) {
 			name:              "Relative link to resource NOT in locality domain",
 			destination:       "./image.png",
 			contentSourcePath: "https://github.com/gardener/gardener/blob/v1.10.0/docs/README.md",
-			wantDestination:   "https://github.com/gardener/gardener/blob/v1.10.0/docs/image.png",
+			wantDestination:   "https://github.com/gardener/gardener/raw/v1.10.0/docs/image.png",
 			wantDownloadURL:   "",
 			wantResourceName:  "",
 			wantErr:           nil,
@@ -128,7 +128,7 @@ func Test_processLink(t *testing.T) {
 			},
 			destination:       "../image.png",
 			contentSourcePath: "https://github.com/gardener/gardener/blob/v1.10.0/docs/README.md",
-			wantDestination:   "https://github.com/gardener/gardener/blob/v1.10.0/image.png",
+			wantDestination:   "https://github.com/gardener/gardener/raw/v1.10.0/image.png",
 			wantDownloadURL:   "",
 			wantResourceName:  "",
 			wantErr:           nil,
@@ -138,7 +138,7 @@ func Test_processLink(t *testing.T) {
 			node:              nodeA,
 			destination:       "https://github.com/owner/repo/blob/master/docs/image.png",
 			contentSourcePath: nodeA.ContentSelectors[0].Source,
-			wantDestination:   "https://github.com/owner/repo/blob/master/docs/image.png",
+			wantDestination:   "https://github.com/owner/repo/raw/master/docs/image.png",
 			wantDownloadURL:   "",
 			wantResourceName:  "",
 			wantErr:           nil,

pkg/reactor/reactor.go

@@ -23,6 +23,7 @@ type Options struct {
 	ResourcesPath                string
 	ResourceDownloadWorkersCount int
 	MarkdownFmt                  bool
+	RewriteEmbedded              bool
 	processors.Processor
 	ResourceDownloadWriter writers.Writer
 	Writer                 writers.Writer
@@ -38,7 +39,7 @@ func NewReactor(o *Options) *Reactor {
 	worker := &DocumentWorker{
 		Writer:               o.Writer,
 		Reader:               &GenericReader{rhRegistry},
-		NodeContentProcessor: NewNodeContentProcessor(o.ResourcesPath, nil, downloadController, o.FailFast, o.MarkdownFmt, rhRegistry),
+		NodeContentProcessor: NewNodeContentProcessor(o.ResourcesPath, nil, downloadController, o.FailFast, o.MarkdownFmt, o.RewriteEmbedded, rhRegistry),
 		Processor:            o.Processor,
 	}
 	docController := NewDocumentController(worker, o.MaxWorkersCount, o.FailFast)

pkg/reactor/reactor_test.go

@@ -144,3 +144,7 @@ func (f *FakeResourceHandler) GetLocalityDomainCandidate(source string) (string,
 func (f *FakeResourceHandler) SetVersion(link, version string) (string, error) {
 	return link, nil
 }
+
+func (f *FakeResourceHandler) GetRawFormatLink(link string) (string, error) {
+	return link, nil
+}

pkg/resourcehandlers/github/github_resource_handler.go

@@ -57,6 +57,7 @@ func TreeEntryToGitHubLocator(treeEntry *github.TreeEntry, shaAlias string) *Res
 		panic(fmt.Sprintf("unexpected resource type %v: %v", treeEntry.GetType(), err))
 	}
 	return &ResourceLocator{
+		"https",
 		host,
 		owner,
 		repo,
@@ -175,7 +176,7 @@ func (c *Cache) Set(path string, entry *ResourceLocator) *ResourceLocator {
 	return entry
 }
 
-// GitHub implements resourcehanlders#ResourceHandler
+// GitHub implements resourcehandlers/ResourceHandler
 type GitHub struct {
 	Client               *github.Client
 	cache                *Cache
@@ -246,7 +247,7 @@ func (gh *GitHub) URLToGitHubLocator(ctx context.Context, urlString string, reso
 	return ghRL, nil
 }
 
-// Accept implements resourcehandlers.ResourceHandler#Accept
+// Accept implements resourcehandlers/ResourceHandler#Accept
 func (gh *GitHub) Accept(uri string) bool {
 	var (
 		url *url.URL
@@ -267,6 +268,7 @@ func (gh *GitHub) Accept(uri string) bool {
 }
 
 // ResolveNodeSelector recursively adds nodes built from tree entries to node
+// ResolveNodeSelector implements resourcehandlers/ResourceHandler#ResolveNodeSelector
 func (gh *GitHub) ResolveNodeSelector(ctx context.Context, node *api.Node) error {
 	var (
 		rl  *ResourceLocator
@@ -285,7 +287,7 @@ func (gh *GitHub) ResolveNodeSelector(ctx context.Context, node *api.Node) error
 	return nil
 }
 
-// Accept implements resourcehandlers.ResourceHandler#Read
+// Read implements resourcehandlers/ResourceHandler#Read
 func (gh *GitHub) Read(ctx context.Context, uri string) ([]byte, error) {
 	var (
 		blob []byte
@@ -315,14 +317,14 @@ func (gh *GitHub) Read(ctx context.Context, uri string) ([]byte, error) {
 			}
 		case Tree:
 			{
-				klog.Warningf("Attempted to read tree object from GitHub: %s. Only wiki and blob URls are supported", rl.String())
+				klog.Warningf("Attempted to read tree object from GitHub: %s. Only wiki pages and blob URLs are supported", rl.String())
 			}
 		}
 	}
 	return blob, err
 }
 
-// Name implements resourcehandlers.ResourceHandler#Name
+// Name implements resourcehandlers/ResourceHandler#Name
 func (gh *GitHub) Name(uri string) string {
 	var (
 		rl  *ResourceLocator
@@ -338,6 +340,7 @@ func (gh *GitHub) Name(uri string) string {
 }
 
 // BuildAbsLink builds the abs link from the source and the relative path
+// Implements resourcehandlers/ResourceHandler#BuildAbsLink
 func (gh *GitHub) BuildAbsLink(source, relPath string) (string, error) {
 	u, err := url.Parse(relPath)
 	if err != nil {
@@ -360,6 +363,7 @@ func (gh *GitHub) BuildAbsLink(source, relPath string) (string, error) {
 
 // GetLocalityDomainCandidate returns the provided source as locality domain candidate
 // parameters suitable for quering reactor/LocalityDomain#PathInLocality
+// Implements resourcehandlers/ResourceHandler#GetLocalityDomainCandidate
 func (gh *GitHub) GetLocalityDomainCandidate(source string) (key, path, version string, err error) {
 	var rl *ResourceLocator
 	if rl, err = parse(source); rl != nil {
@@ -376,6 +380,7 @@ func (gh *GitHub) GetLocalityDomainCandidate(source string) (key, path, version
 
 // SetVersion replaces the version segment in the path of GitHub URLs if
 // applicable or returns the original URL unchanged if not.
+// Implements resourcehandlers/ResourceHandler#SetVersion
 func (gh *GitHub) SetVersion(absLink, version string) (string, error) {
 	var (
 		rl  *ResourceLocator
@@ -396,3 +401,18 @@ func (gh *GitHub) SetVersion(absLink, version string) (string, error) {
 
 	return absLink, nil
 }
+
+// GetRawFormatLink implements ResourceHandler#GetRawFormatLink
+func (gh *GitHub) GetRawFormatLink(absLink string) (string, error) {
+	var (
+		rl  *ResourceLocator
+		err error
+	)
+	if rl, err = parse(absLink); err != nil {
+		return "", err
+	}
+	if l := rl.GetRaw(); len(l) > 0 {
+		return l, nil
+	}
+	return absLink, nil
+}

pkg/resourcehandlers/github/github_resource_handler_test.go

@@ -64,6 +64,7 @@ func setup() (client *github.Client, mux *http.ServeMux, serverURL string, teard
 
 func TestUrlToGitHubLocator(t *testing.T) {
 	ghrl1 := &ResourceLocator{
+		"https",
 		"github.com",
 		"gardener",
 		"gardener",
@@ -73,6 +74,7 @@ func TestUrlToGitHubLocator(t *testing.T) {
 		"master",
 	}
 	ghrl2 := &ResourceLocator{
+		"https",
 		"github.com",
 		"gardener",
 		"gardener",
@@ -274,6 +276,7 @@ func TestResolveNodeSelector(t *testing.T) {
 
 func TestName(t *testing.T) {
 	ghrl1 := &ResourceLocator{
+		"https",
 		"github.com",
 		"gardener",
 		"gardener",
@@ -283,6 +286,7 @@ func TestName(t *testing.T) {
 		"",
 	}
 	ghrl2 := &ResourceLocator{
+		"https",
 		"github.com",
 		"gardener",
 		"gardener",
@@ -351,6 +355,7 @@ func TestRead(t *testing.T) {
 			&Cache{
 				cache: map[string]*ResourceLocator{
 					"https://github.com/gardener/gardener/blob/master/docs/README.md": &ResourceLocator{
+						"https",
 						"github.com",
 						"gardener",
 						"gardener",