Merge pull request #329 from Kostov6/minor-changes

Minor changes

cmd/app/exec.go

@@ -67,10 +67,7 @@ func exec(ctx context.Context) error {
 	if err != nil {
 		return err
 	}
-	if !config.ValidateLinks {
-		v = nil
-	}
-	docProcessor, docTasks, err := document.New(config.DocumentWorkersCount, config.FailFast, reactorWG, documentNodes, config.ResourcesPath, dScheduler, v, rhRegistry, config.Hugo, config.Writer)
+	docProcessor, docTasks, err := document.New(config.DocumentWorkersCount, config.FailFast, reactorWG, documentNodes, config.ResourcesPath, dScheduler, v, rhRegistry, config.Hugo, config.Writer, config.SkipLinkValidation)
 	if err != nil {
 		return err
 	}

cmd/app/flags.go

@@ -76,9 +76,9 @@ func configureFlags(command *cobra.Command) {
 		"Supported content format extensions (exampel: .md)")
 	_ = vip.BindPFlag("extracted-files-formats", command.Flags().Lookup("extracted-files-formats"))
 
-	command.Flags().Bool("validate-links", true,
-		"Links should be validated")
-	_ = vip.BindPFlag("validate-links", command.Flags().Lookup("validate-links"))
+	command.Flags().Bool("skip-link-validation", false,
+		"Links validation will be skipped")
+	_ = vip.BindPFlag("skip-link-validation", command.Flags().Lookup("skip-link-validation"))
 
 	command.Flags().StringSlice("hosts-to-report", []string{},
 		"When a link has a host from the given array it will get reported")

cmd/app/types.go

@@ -24,8 +24,8 @@ type Options struct {
 	DryRun                       bool     `mapstructure:"dry-run"`
 	Resolve                      bool     `mapstructure:"resolve"`
 	ExtractedFilesFormats        []string `mapstructure:"extracted-files-formats"`
-	ValidateLinks                bool     `mapstructure:"validate-links"`
 	HostsToReport                []string `mapstructure:"hosts-to-report"`
+	SkipLinkValidation           bool     `mapstructure:"skip-link-validation"`
 }
 
 // Writers struct that collects all the writesr

pkg/manifest/manifest.go

@@ -348,7 +348,9 @@ func propagateFrontmatter(node *Node, parent *Node, manifest *Node, _ registry.I
 	if parent != nil {
 		newFM := map[string]interface{}{}
 		for k, v := range parent.Frontmatter {
-			newFM[k] = v
+			if k != "aliases" {
+				newFM[k] = v
+			}
 		}
 		for k, v := range node.Frontmatter {
 			newFM[k] = v
@@ -358,41 +360,52 @@ func propagateFrontmatter(node *Node, parent *Node, manifest *Node, _ registry.I
 	return nil
 }
 
+func propagateSkipValidation(node *Node, parent *Node, manifest *Node, _ registry.Interface) error {
+	if parent != nil && parent.SkipValidation {
+		node.SkipValidation = parent.SkipValidation
+	}
+	return nil
+}
+
 func setParent(node *Node, parent *Node, _ *Node, _ registry.Interface) error {
 	node.parent = parent
 	return nil
 }
 
-// func calculateAliases(node *Node, parent *Node, _ *Node, _ registry.Interface) error {
-// 	var (
-// 		nodeAliases  []interface{}
-// 		childAliases []interface{}
-// 		formatted    bool
-// 	)
-// 	if nodeAliases, formatted = node.Frontmatter["aliases"].([]interface{}); node.Frontmatter != nil && node.Frontmatter["aliases"] != nil && !formatted {
-// 		return fmt.Errorf("node X \n\n%s\n has invalid alias format", node)
-// 	}
-// 	for _, nodeAlias := range nodeAliases {
-// 		for _, child := range node.Structure {
-// 			if child.Frontmatter == nil {
-// 				child.Frontmatter = map[string]interface{}{}
-// 			}
-// 			if child.Frontmatter["aliases"] == nil {
-// 				child.Frontmatter["aliases"] = []interface{}{}
-// 			}
-// 			if childAliases, formatted = child.Frontmatter["aliases"].([]interface{}); !formatted {
-// 				return fmt.Errorf("node \n\n%s\n has invalid alias format", child)
-// 			}
-// 			finalAlias := strings.TrimSuffix(child.Name(), ".md") + "/"
-// 			if child.Name() == "_index.md" {
-// 				finalAlias = ""
-// 			}
-// 			childAliases = append(childAliases, fmt.Sprintf("%s", nodeAlias)+"/"+finalAlias)
-// 			child.Frontmatter["aliases"] = childAliases
-// 		}
-// 	}
-// 	return nil
-// }
+func calculateAliases(node *Node, parent *Node, _ *Node, _ registry.Interface) error {
+	var (
+		nodeAliases  []interface{}
+		childAliases []interface{}
+		formatted    bool
+	)
+	if nodeAliases, formatted = node.Frontmatter["aliases"].([]interface{}); node.Frontmatter != nil && node.Frontmatter["aliases"] != nil && !formatted {
+		return fmt.Errorf("node X \n\n%s\n has invalid alias format", node)
+	}
+	for _, nodeAliasI := range nodeAliases {
+		for _, child := range node.Structure {
+			if child.Frontmatter == nil {
+				child.Frontmatter = map[string]interface{}{}
+			}
+			if child.Frontmatter["aliases"] == nil {
+				child.Frontmatter["aliases"] = []interface{}{}
+			}
+			if childAliases, formatted = child.Frontmatter["aliases"].([]interface{}); !formatted {
+				return fmt.Errorf("node \n\n%s\n has invalid alias format", child)
+			}
+			childAliasSuffix := strings.TrimSuffix(child.Name(), ".md")
+			if child.Name() == "_index.md" {
+				childAliasSuffix = ""
+			}
+			nodeAlias := fmt.Sprintf("%s", nodeAliasI)
+			if !strings.HasPrefix(nodeAlias, "/") {
+				return fmt.Errorf("there is a node with name %s that has an relative alias %s", node.Name(), nodeAlias)
+			}
+			childAliases = append(childAliases, path.Join(nodeAlias, childAliasSuffix)+"/")
+			child.Frontmatter["aliases"] = childAliases
+		}
+	}
+	return nil
+}
 
 // ResolveManifest collects files in FileCollector from a given url and resourcehandlers.FileSource
 func ResolveManifest(url string, r registry.Interface) ([]*Node, error) {
@@ -417,6 +430,8 @@ func ResolveManifest(url string, r registry.Interface) ([]*Node, error) {
 		calculatePath,
 		setParent,
 		propagateFrontmatter,
+		propagateSkipValidation,
+		calculateAliases,
 	)
 	if err != nil {
 		return nil, err

pkg/manifest/manifest_test.go

@@ -66,6 +66,7 @@ var _ = Describe("Manifest test", func() {
 		Entry("covering directory merges", "merging"),
 		Entry("covering manifest use cases", "manifest"),
 		Entry("covering multisource", "multisource"),
+		Entry("covering aliases", "aliases"),
 	)
 
 	DescribeTable("Errors",

pkg/manifest/node.go

@@ -22,7 +22,7 @@ type Node struct {
 	FilesTreeType `yaml:",inline"`
 
 	// Properties of the node
-	Properties map[string]interface{} `yaml:"properties,omitempty"`
+	SkipValidation bool `yaml:"skipValidation,omitempty"`
 	// Frontmatter of the node
 	Frontmatter map[string]interface{} `yaml:"frontmatter,omitempty"`
 	// Type of node

pkg/manifest/tests/manifests/aliases.yaml

@@ -0,0 +1,29 @@
+structure:
+- dir: root
+  structure:
+  - dir: topic
+    structure:
+    - dir: new_section_1
+      frontmatter:
+        aliases:
+        - "/root2/rebase"
+        - "/root3/rebase_slash/"
+      structure:
+      - file: /contents/README.md
+      - dir: subsection
+        structure:
+        - file: /contents/website/blog/2024/_index.md
+        - file: /contents/blogs/2024/foo.md
+          frontmatter:
+            aliases:
+            - "/root4/alias/"
+  - file: /contents/blogs/2024/two.md
+    frontmatter:
+      aliases:
+      - "/root4/normal/alias/"
+  - dir: architecture
+    frontmatter:
+      aliases:
+      - "/root"
+    structure:
+  - fileTree: /contents/docs

pkg/manifest/tests/results/aliases.yaml

@@ -0,0 +1,46 @@
+- file: README.md
+  type: file
+  source: https://github.com/gardener/docforge/blob/master/contents/README.md
+  path: root/topic/new_section_1
+  frontmatter:
+    aliases:
+    - "/root2/rebase/README/"
+    - "/root3/rebase_slash/README/"
+- file: _index.md
+  type: file
+  source: https://github.com/gardener/docforge/blob/master/contents/website/blog/2024/_index.md
+  path: root/topic/new_section_1/subsection
+  frontmatter:
+    aliases:
+    - "/root2/rebase/subsection/"
+    - "/root3/rebase_slash/subsection/"
+- file: foo.md
+  type: file
+  source: https://github.com/gardener/docforge/blob/master/contents/blogs/2024/foo.md
+  path: root/topic/new_section_1/subsection
+  frontmatter:
+    aliases:
+    - "/root4/alias/"
+    - "/root2/rebase/subsection/foo/"
+    - "/root3/rebase_slash/subsection/foo/"
+- file: two.md
+  type: file
+  source: https://github.com/gardener/docforge/blob/master/contents/blogs/2024/two.md
+  path: root
+  frontmatter:
+    aliases:
+    - "/root4/normal/alias/"
+- file: _index.md
+  type: file
+  source: https://github.com/gardener/docforge/blob/master/contents/docs/architecture/_index.md
+  path: root/architecture
+  frontmatter:
+    aliases:
+    - "/root/"
+- file: concept.md
+  type: file
+  source: https://github.com/gardener/docforge/blob/master/contents/docs/architecture/concept.md
+  path: root/architecture
+  frontmatter:
+    aliases:
+    - "/root/concept/"

pkg/registry/registry.go

@@ -49,55 +49,58 @@ func NewRegistry(resourcerepoHosts ...repositoryhost.Interface) Interface {
 }
 
 func (r *registry) Client(url string) httpclient.Client {
-	rh, _, err := r.repositoryHost(url)
+	rh, _, err := r.anyRepositoryHost(url)
 	if err != nil {
 		return http.DefaultClient
 	}
 	return rh.GetClient()
 }
 
 func (r *registry) Tree(resourceURL string) ([]string, error) {
-	rh, url, err := r.repositoryHost(resourceURL)
+	rh, url, err := r.anyRepositoryHost(resourceURL)
 	if err != nil {
 		return []string{}, err
 	}
 	return rh.Tree(*url)
 }
 
 func (r *registry) Read(ctx context.Context, resourceURL string) ([]byte, error) {
-	rh, url, err := r.repositoryHost(resourceURL)
+	rh, url, err := r.anyRepositoryHost(resourceURL)
 	if err != nil {
 		return []byte{}, err
 	}
 	return rh.Read(ctx, *url)
 }
 
 func (r *registry) ResolveRelativeLink(source string, relativeLink string) (string, error) {
-	rh, url, err := r.repositoryHost(source)
+	rh, url, err := r.anyRepositoryHost(source)
 	if err != nil {
 		return "", err
 	}
 	return rh.ResolveRelativeLink(*url, relativeLink)
 }
 
 func (r *registry) ReadGitInfo(ctx context.Context, resourceURL string) ([]byte, error) {
-	rh, url, err := r.gitInfoRepositoryHost(resourceURL)
+	rh, url, err := r.githubRepositoryHost(resourceURL)
 	if err != nil {
 		return []byte{}, err
 	}
 	return repositoryhost.ReadGitInfo(ctx, rh.Repositories(), *url)
 }
 
 func (r *registry) LoadRepository(ctx context.Context, resourceURL string) error {
-	rh, err := r.get(resourceURL)
+	rh, err := r.acceptGithubRH(resourceURL)
 	if err != nil {
+		if err.Error() == fmt.Sprintf("no sutiable repository host for %s", resourceURL) {
+			return nil
+		}
 		return err
 	}
 	return rh.LoadRepository(ctx, resourceURL)
 }
 
-func (r *registry) repositoryHost(resourceURL string) (repositoryhost.Interface, *repositoryhost.URL, error) {
-	rh, err := r.get(resourceURL)
+func (r *registry) anyRepositoryHost(resourceURL string) (repositoryhost.Interface, *repositoryhost.URL, error) {
+	rh, err := r.acceptAnyRH(resourceURL)
 	if err != nil {
 		return nil, nil, err
 	}
@@ -109,12 +112,12 @@ func (r *registry) repositoryHost(resourceURL string) (repositoryhost.Interface,
 }
 
 func (r *registry) ResourceURL(resourceURL string) (*repositoryhost.URL, error) {
-	_, url, err := r.repositoryHost(resourceURL)
+	_, url, err := r.anyRepositoryHost(resourceURL)
 	return url, err
 }
 
-func (r *registry) gitInfoRepositoryHost(resourceURL string) (repositoryhost.Interface, *repositoryhost.URL, error) {
-	rh, err := r.getGitInfo(resourceURL)
+func (r *registry) githubRepositoryHost(resourceURL string) (repositoryhost.Interface, *repositoryhost.URL, error) {
+	rh, err := r.acceptGithubRH(resourceURL)
 	if err != nil {
 		return nil, nil, err
 	}
@@ -126,7 +129,7 @@ func (r *registry) gitInfoRepositoryHost(resourceURL string) (repositoryhost.Int
 	return rh, url, nil
 }
 
-func (r *registry) get(uri string) (repositoryhost.Interface, error) {
+func (r *registry) acceptAnyRH(uri string) (repositoryhost.Interface, error) {
 	for _, h := range r.repoHosts {
 		if h.Accept(uri) {
 			return h, nil
@@ -135,7 +138,7 @@ func (r *registry) get(uri string) (repositoryhost.Interface, error) {
 	return nil, fmt.Errorf("no sutiable repository host for %s", uri)
 }
 
-func (r *registry) getGitInfo(uri string) (repositoryhost.Interface, error) {
+func (r *registry) acceptGithubRH(uri string) (repositoryhost.Interface, error) {
 	for _, h := range r.repoHosts {
 		if h.Repositories() != nil && h.Accept(uri) {
 			return h, nil

pkg/registry/repositoryhost/github_http_cache_test.go

@@ -98,18 +98,6 @@ var _ = Describe("Github cache test", func() {
 	git.GetTreeReturns(&tree, nil, nil)
 	ghc.LoadRepository(context.TODO(), "https://github.com/gardener/docforge/blob/master/README.md")
 
-	Describe("#GetRateLimit", func() {
-		BeforeEach(func() {
-			rls.RateLimitsReturns(nil, nil, errors.New("yataa error"))
-		})
-
-		It("return correct rate limit", func() {
-			_, _, _, err := ghc.GetRateLimit(context.TODO())
-			Expect(err).To(Equal(errors.New("yataa error")))
-
-		})
-	})
-
 	testRepositoryHost(ghc)
 
 	It("repository updated after loading", func() {

pkg/workers/document/document_worker.go

@@ -39,8 +39,9 @@ type Worker struct {
 
 	resourcesRoot string
 
-	repositoryhosts registry.Interface
-	hugo            hugo.Hugo
+	repositoryhosts    registry.Interface
+	hugo               hugo.Hugo
+	skipLinkValidation bool
 }
 
 // docContent defines a document content
@@ -51,7 +52,7 @@ type docContent struct {
 }
 
 // NewDocumentWorker creates Worker objects
-func NewDocumentWorker(resourcesRoot string, downloader downloader.Interface, validator linkvalidator.Interface, linkResolver linkresolver.Interface, rh registry.Interface, hugo hugo.Hugo, writer writers.Writer) *Worker {
+func NewDocumentWorker(resourcesRoot string, downloader downloader.Interface, validator linkvalidator.Interface, linkResolver linkresolver.Interface, rh registry.Interface, hugo hugo.Hugo, writer writers.Writer, skipLinkValidation bool) *Worker {
 	return &Worker{
 		linkResolver,
 		downloader,
@@ -60,6 +61,7 @@ func NewDocumentWorker(resourcesRoot string, downloader downloader.Interface, va
 		resourcesRoot,
 		rh,
 		hugo,
+		skipLinkValidation,
 	}
 }
 
@@ -195,7 +197,9 @@ func (d *linkResolverTask) resolveLink(dest string, isEmbeddable bool) (string,
 	if url.IsAbs() {
 		if _, err = d.repositoryhosts.ResourceURL(dest); err != nil {
 			// absolute link that is not referencing any documentation page
-			d.validator.ValidateLink(dest, d.source)
+			if !d.node.SkipValidation && !d.skipLinkValidation {
+				d.validator.ValidateLink(dest, d.source)
+			}
 			return dest, nil
 		}
 	}