+ node-level links scope, exclude/include

pkg/api/parser_test.go

@@ -17,10 +17,15 @@ package api
 
 import (
 	"fmt"
+	"io/ioutil"
+	"path/filepath"
 	"testing"
+
+	"github.com/stretchr/testify/assert"
 )
 
-var b = []byte(`root:
+var b = []byte(`
+root:
   name: root
   nodes:
   - name: node_1
@@ -31,10 +36,21 @@ var b = []byte(`root:
     - source: https://a.com
     properties:
       "custom_key": custom_value
+    localityDomain:
+      github.com/gardener/gardener:
+        exclude:
+        - a
     nodes:
     - name: subnode
       contentSelectors:	
-      - source: path/a`)
+      - source: path/a
+localityDomain:
+  github.com/gardener/gardener:
+    version: v1.10.0
+    path: gardener/gardener/docs
+    LinkSubstitutes:
+      a: b
+`)
 
 func traverse(node *Node) {
 	fmt.Printf("%++v \n", node)
@@ -140,3 +156,62 @@ func TestMe(t *testing.T) {
 		return
 	}
 }
+
+func TestFile(t *testing.T) {
+	var (
+		blob []byte
+		err  error
+		got  *Documentation
+	)
+	expected := &Documentation{
+		Root: &Node{
+			Name: "00",
+			Nodes: []*Node{
+				&Node{
+					Name: "01",
+					ContentSelectors: []ContentSelector{
+						ContentSelector{
+							Source: "https://github.com/gardener/gardener/blob/master/docs/concepts/gardenlet.md",
+						},
+					},
+					LocalityDomain: LocalityDomain{
+						"github.com/gardener/gardener": &LocalityDomainValue{
+							Version: "v1.11.1",
+							Path:    "gardener/gardener",
+							LinksMatchers: LinksMatchers{
+								Exclude: []string{
+									"example",
+								},
+							},
+						},
+					},
+				},
+				&Node{
+					Name: "02",
+					ContentSelectors: []ContentSelector{
+						ContentSelector{
+							Source: "https://github.com/gardener/gardener/blob/master/docs/deployment/deploy_gardenlet.md",
+						},
+					},
+				},
+			},
+		},
+		LocalityDomain: LocalityDomain{
+			"github.com/gardener/gardener": &LocalityDomainValue{
+				Version: "v1.10.0",
+				Path:    "gardener/gardener",
+			},
+		},
+	}
+
+	if blob, err = ioutil.ReadFile(filepath.Join("testdata", "parse_test_00.yaml")); err != nil {
+		t.Fatalf(err.Error())
+	}
+	got, err = Parse(blob)
+	if err != nil {
+		t.Errorf("%v\n", err)
+	}
+	if got != expected {
+		assert.Equal(t, expected, got)
+	}
+}

pkg/api/testdata/parse_test_00.yaml

@@ -0,0 +1,19 @@
+root:
+  name: 00
+  nodes:
+  - name: 01
+    contentSelectors:
+      - source: https://github.com/gardener/gardener/blob/master/docs/concepts/gardenlet.md
+    localityDomain:
+      github.com/gardener/gardener:
+        version: v1.11.1
+        path: gardener/gardener
+        exclude:
+        - example
+  - name: 02
+    contentSelectors:
+      - source: https://github.com/gardener/gardener/blob/master/docs/deployment/deploy_gardenlet.md
+localityDomain:
+  github.com/gardener/gardener:
+    version: v1.10.0
+    path: gardener/gardener

pkg/api/types.go

@@ -29,7 +29,8 @@ type Documentation struct {
 	//
 	// Note: WiP - proposed, not implemented yet.
 	Variables map[string]*Node `yaml:"variables,omitempty"`
-	// LocalityDomain is the range
+	// LocalityDomain defines the scope of the downloadable resources
+	// for this structure
 	LocalityDomain LocalityDomain `yaml:"localityDomain,omitempty"`
 }
 
@@ -86,10 +87,8 @@ type Node struct {
 	// and the serialization of the Node. For example the properyies member could be
 	// used to set the front-matter to markdowns for front-matter aware builders such
 	// as Hugo.
-	Properties map[string]interface{} `yaml:"properties,omitempty"`
-	// Links is an optional tunning of the mechanisms for processing
-	// links applicable to document nodes.
-	Links *Links `yaml:"links,omitempty"`
+	Properties     map[string]interface{} `yaml:"properties,omitempty"`
+	LocalityDomain `yaml:"localityDomain,omitempty"`
 }
 
 // NodeSelector is an specification for selecting subnodes (children) for a node.
@@ -134,22 +133,8 @@ type ContentSelector struct {
 	Selector *string `yaml:"selector,omitempty"`
 }
 
-// Links defines rules for links handling
-type Links struct {
-	// Substitutes is an optional map of links and their
-	// substitutions. Use it to override the default links handling:
-	// - An empty substritution string ("") can be used to remove
-	//   a link mark (applies only to markdown links and images).
-	// - A fixed string that will replace the whole original link.
-	// - An expression with substitution variables can be used
-	//   to change the default pattern for generating donwloaded resouce
-	//   names, which is $uuid. That applies only to downloaded resouces.
-	//   The supported variables are
-	//   - $name: the original name of the resouce
-	//   - $path: the original path fo the resource (may be empty)
-	//   - $uuid: the identifier generated for the downloaded resource
-	//   Example: $name-$uuid
-	Substitutes map[string]string `yaml:"substitutes,omitempty"`
+// LinksMatchers defines links exclusion/inclusion patterns
+type LinksMatchers struct {
 	// Include is a list of regular expressions that will be matched to every
 	// link that is candidate for download to determine whether it is
 	// eligible. The links to match are absolute.
@@ -174,12 +159,46 @@ type Links struct {
 // are always part of the locality domain. Other
 // resources referenced by those documents are checked
 // against the path hierarchy of locality domain
-// entries to determine hwo they will be processed.
+// entries to determine how they will be processed.
 type LocalityDomain map[string]*LocalityDomainValue
 
 // LocalityDomainValue encapsulates the memebers of a
 // LocalityDomain entry value
 type LocalityDomainValue struct {
+	// Version sets the version of the resources that will
+	// be referenced in this domain. Download targets and
+	// absolute links in documents referenced by the structure
+	// will be rewritten to match this version
 	Version string `yaml:"version"`
-	Path    string `yaml:"path"`
+	// Path is the relative path inside a domain that contains
+	// resources considered 'local' that will be downloaded.
+	Path          string `yaml:"path"`
+	LinksMatchers `yaml:",inline"`
+	// LinkSubstitutes is an optional map of links and their
+	// substitutions. Use it to override the default handling of those
+	// links in documents in this locality domain:
+	// - An empty substitution string ("") removes a link markdown
+	//   turning. It leaves only its text component in the document
+	//   for links and nothing for images.
+	//   This applies only to markdown for links and images.
+	// - A fixed string that will replace the whole original link
+	//   destination.
+	LinkSubstitutes Substitutes
+	// DownloadSubstitutes is an optional map of resource names in this
+	// locality domain and their substitutions. Use it to override the
+	// default downloads naming:
+	// - An exact download name mapped to a download resource will be used
+	//   to name that resources when downloaded.
+	// - An expression with substitution variables can be used
+	//   to change the default pattern for generating donwloaded resouce
+	//   names, which is $uuid.
+	//   The supported variables are:
+	//   - $name: the original name of the resouce
+	//   - $path: the original path of the resource in this domain (may be empty)
+	//   - $uuid: the identifier generated for the downloaded resource
+	//   Example expression: $name-$uuid
+	DownloadSubstitutes Substitutes
 }
+
+// Substitutes is map of ...
+type Substitutes map[string]string

pkg/reactor/content_processor.go

@@ -149,7 +149,7 @@ func (c *NodeContentProcessor) reconcileHTMLLinks(ctx context.Context, docNode *
 
 // returns destination, downloadURL, resourceName, err
 func (c *NodeContentProcessor) processLink(ctx context.Context, node *api.Node, destination string, contentSourcePath string) (string, string, string, error) {
-	if strings.HasPrefix(destination, "#") {
+	if strings.HasPrefix(destination, "#") || strings.HasPrefix(destination, "mailto:") {
 		return destination, "", "", nil
 	}
 
@@ -167,39 +167,14 @@ func (c *NodeContentProcessor) processLink(ctx context.Context, node *api.Node,
 		return "", "", "", err
 	}
 	_a := absLink
-	var (
-		include, exclude bool
-	)
-	// check if the links is not eligible by explicit exclude
-	if node.Links != nil && len(node.Links.Exclude) > 0 {
-		for _, rx := range node.Links.Exclude {
-			if exclude, err = regexp.MatchString(rx, absLink); err != nil {
-				klog.V(6).Infof("[%s] exclude pattern match %s failed for %s\n", contentSourcePath, node.Links.Exclude, absLink)
-			}
-			if exclude {
-				break
-			}
-		}
+	ld := c.localityDomain
+	if node != nil {
+		ld = resolveLocalityDomain(node, c.localityDomain)
 	}
-	absLink, inLD := c.localityDomain.MatchPathInLocality(absLink, c.ResourceHandlers)
+	absLink, inLD := ld.MatchPathInLocality(absLink, c.ResourceHandlers)
 	if _a != absLink {
 		klog.V(6).Infof("[%s] Link converted %s -> %s\n", contentSourcePath, _a, absLink)
 	}
-	// check if the links is eligible by explicit include
-	if node.Links != nil && len(node.Links.Include) > 0 {
-		for _, rx := range node.Links.Include {
-			if include, err = regexp.MatchString(rx, absLink); err != nil {
-				klog.V(6).Infof("[%s] exclude pattern match %s failed for %s\n", contentSourcePath, node.Links.Exclude, absLink)
-			}
-			if include {
-				break
-			}
-		}
-		exclude = !include
-	}
-	if exclude {
-		return absLink, "", "", nil
-	}
 	// Links to other documents are enforced relative when
 	// linking documents from the node structure.
 	// Links to other documents are changed to match the linking
@@ -221,7 +196,7 @@ func (c *NodeContentProcessor) processLink(ctx context.Context, node *api.Node,
 	// Links to resources are assessed for download eligibility
 	// and if applicable their destination is updated as relative
 	// path to predefined location for resources
-	if absLink != "" && (inLD || include) {
+	if absLink != "" && inLD { //(inLD || include)
 		resourceName := c.generateResourceName(absLink)
 		_d := destination
 		destination = buildDestination(node, resourceName, c.resourcesRoot)

pkg/reactor/content_processor_test.go

@@ -51,6 +51,15 @@ func Test_processLink(t *testing.T) {
 			wantResourceName:  "",
 			wantErr:           nil,
 		},
+		{
+			name:              "mailto protocol is not processed",
+			destination:       "mailto:[email protected]",
+			contentSourcePath: "",
+			wantDestination:   "mailto:[email protected]",
+			wantDownloadURL:   "",
+			wantResourceName:  "",
+			wantErr:           nil,
+		},
 		{
 			name:              "Absolute links to releases not processed",
 			destination:       "https://github.com/gardener/gardener/releases/tag/v1.4.0",
@@ -99,6 +108,10 @@ func Test_processLink(t *testing.T) {
 					"github.com/gardener/gardener": &localityDomainValue{
 						"v1.10.0",
 						"gardener/gardener/docs",
+						nil,
+						nil,
+						nil,
+						nil,
 					},
 				}
 			},
@@ -149,6 +162,10 @@ func Test_processLink(t *testing.T) {
 					"github.com/gardener/gardener": &localityDomainValue{
 						"v1.10.0",
 						"gardener/gardener/docs",
+						nil,
+						nil,
+						nil,
+						nil,
 					},
 				}
 			},
@@ -178,6 +195,10 @@ func Test_processLink(t *testing.T) {
 					"github.com/gardener/gardener": &localityDomainValue{
 						"v1.10.0",
 						"gardener/gardener/docs",
+						nil,
+						nil,
+						nil,
+						nil,
 					},
 				}
 			},
@@ -206,6 +227,10 @@ func Test_processLink(t *testing.T) {
 					"github.com/gardener/gardener": &localityDomainValue{
 						"v1.10.0",
 						"gardener/gardener/docs",
+						nil,
+						nil,
+						nil,
+						nil,
 					},
 				}
 			},
@@ -225,6 +250,10 @@ func Test_processLink(t *testing.T) {
 					"github.com/gardener/gardener": &localityDomainValue{
 						"v1.10.0",
 						"gardener/gardener/docs",
+						nil,
+						nil,
+						nil,
+						nil,
 					},
 				}
 			},
@@ -243,6 +272,10 @@ func Test_processLink(t *testing.T) {
 					"github.com/gardener/gardener": &localityDomainValue{
 						"v1.10.0",
 						"gardener/gardener/docs",
+						nil,
+						nil,
+						nil,
+						nil,
 					},
 				}
 			},

pkg/reactor/document_worker_test.go

@@ -41,7 +41,7 @@ func (p *TestProcessor) Process(documentBlob []byte, node *api.Node) ([]byte, er
 }
 
 func TestDocumentWorkerWork(t *testing.T) {
-	testOutput := "#Heading 1"
+	testOutput := "#Heading 1\n"
 	rhRegistry := resourcehandlers.NewRegistry(&FakeResourceHandler{})
 	testworker := &DocumentWorker{
 		&TestWriter{