From ca37da4282cf59ec34d4ca4f329d7608611b5240 Mon Sep 17 00:00:00 2001 From: Leon Hudak <33522493+leohhhn@users.noreply.github.com> Date: Wed, 19 Jun 2024 19:36:36 +0200 Subject: [PATCH] docs: add 404 link checker / linter (#2394) ## Description This PR cherrypicks the 404 link checker / linter from #2125.
Contributors' checklist... - [x] Added new tests, or not needed, or not feasible - [x] Provided an example (e.g. screenshot) to aid review or the PR is self-explanatory - [x] Updated the official documentation or not needed - [x] No breaking changes were made, or a `BREAKING CHANGE: xxx` message was included in the description - [x] Added references to related issues and PRs - [x] Provided any useful hints for running manual tests - [x] Added new benchmarks to [generated graphs](https://gnoland.github.io/benchmarks), if any. More info [here](https://github.com/gnolang/gno/blob/master/.benchmarks/README.md).
--- .github/workflows/docs-404-checker.yml | 31 ++++ docs/Makefile | 12 ++ docs/concepts/effective-gno.md | 2 +- docs/concepts/packages.md | 4 +- docs/concepts/tendermint2.md | 4 +- docs/concepts/testnets.md | 2 +- docs/how-to-guides/simple-library.md | 2 +- docs/reference/network-config.md | 12 +- misc/docs-linter/go.mod | 19 +++ misc/docs-linter/go.sum | 22 +++ misc/docs-linter/main.go | 218 +++++++++++++++++++++++++ misc/docs-linter/main_test.go | 126 ++++++++++++++ 12 files changed, 440 insertions(+), 14 deletions(-) create mode 100644 .github/workflows/docs-404-checker.yml create mode 100644 docs/Makefile create mode 100644 misc/docs-linter/go.mod create mode 100644 misc/docs-linter/go.sum create mode 100644 misc/docs-linter/main.go create mode 100644 misc/docs-linter/main_test.go diff --git a/.github/workflows/docs-404-checker.yml b/.github/workflows/docs-404-checker.yml new file mode 100644 index 00000000000..0fa8985366c --- /dev/null +++ b/.github/workflows/docs-404-checker.yml @@ -0,0 +1,31 @@ +name: "docs / 404 checker" + +on: + push: + paths: + - master + pull_request: + paths: + - "docs/**" + +jobs: + build: + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v2 + + - name: Set up Go + uses: actions/setup-go@v3 + with: + go-version: '1.21' + + - name: Install dependencies + run: go mod download + + - name: Build docs + run: make -C docs/ build + + - name: Run linter + run: make -C docs/ lint \ No newline at end of file diff --git a/docs/Makefile b/docs/Makefile new file mode 100644 index 00000000000..e5bf557ef75 --- /dev/null +++ b/docs/Makefile @@ -0,0 +1,12 @@ +all: build lint + +# Build the linter +build: + cd ../misc/docs-linter && go build -o ./build/ + +# Run the linter for the docs/ folder +lint: + ../misc/docs-linter/build/linter -path . + +clean: + rm -rf ../misc/docs-linter/build \ No newline at end of file diff --git a/docs/concepts/effective-gno.md b/docs/concepts/effective-gno.md index d08a9089487..8e589f41845 100644 --- a/docs/concepts/effective-gno.md +++ b/docs/concepts/effective-gno.md @@ -679,7 +679,7 @@ For example, if you're creating a coin for cross-chain transfers, Coins are your best bet. They're IBC-ready and their strict rules offer top-notch security. -Read about how to use the Banker module [here](stdlibs/banker). +Read about how to use the Banker module [here](stdlibs/banker.md). #### GRC20 tokens diff --git a/docs/concepts/packages.md b/docs/concepts/packages.md index 79f54d4f59e..cd3e2ace96a 100644 --- a/docs/concepts/packages.md +++ b/docs/concepts/packages.md @@ -12,7 +12,7 @@ The full list of pre-deployed available packages can be found under the [demo pa In Go, the classic key/value data type is represented by the `map` construct. However, while Gno also supports the use of `map`, it is not a viable option as it lacks determinism due to its non-sequential nature. -To address this issue, Gno implements the [AVL Tree](https://en.wikipedia.org/wiki/AVL\_tree) (Adelson-Velsky-Landis Tree) as a solution. The AVL Tree is a self-balancing binary search tree. +To address this issue, Gno implements the [AVL Tree](https://en.wikipedia.org/wiki/AVL_tree) (Adelson-Velsky-Landis Tree) as a solution. The AVL Tree is a self-balancing binary search tree. The `avl` package comprises a set of functions that can manipulate the leaves and nodes of the AVL Tree. @@ -72,7 +72,7 @@ func IsApprovedForAll(owner, operator std.Address) bool * `OwnerOf`: Returns the `owner`'s address of a token specified by its `TokenID`. * `SafeTransferFrom`: Equivalent to the `TransferFrom` function of `grc20`. * The `Safe` prefix indicates that the function runs a check to ensure that the `to` address is a valid address that can receive tokens. - * As you can see from the [code](https://github.com/gnolang/gno/blob/master/examples/gno.land/p/demo/grc/grc721/basic\_nft.gno#L341), the concept of `Safe` has yet to be implemented. + * As you can see from the [code](https://github.com/gnolang/gno/blob/master/examples/gno.land/p/demo/grc/grc721/basic_nft.gno#L341), the concept of `Safe` has yet to be implemented. * `SetApprovalForAll`: Approves all tokens owned by the `owner` to an `operator`. * You may not set multiple `operator`s. * `GetApproved`: Returns the `address` of the `operator` for a token, specified with its `ID`. diff --git a/docs/concepts/tendermint2.md b/docs/concepts/tendermint2.md index a6004606a78..4dd43b0819e 100644 --- a/docs/concepts/tendermint2.md +++ b/docs/concepts/tendermint2.md @@ -34,9 +34,7 @@ on https://github.com/tendermint/tendermint2.** proto3 for encoding/decoding optimization through protoc. - MISSION: be the basis for improving the encoding standard from proto3, because proto3 length-prefixing is slow, and we need "proto4" or "amino2". - - LOOK at the auto-generated proto files! - https://github.com/gnolang/gno/blob/master/pkgs/bft/consensus/types/cstypes.proto - for example. + - LOOK at the [auto-generated proto files](https://github.com/gnolang/gno/blob/master/tm2/pkg/bft/consensus/consensus.proto)! - There was work to remove this from the CosmosSDK because Amino wasn't ready, but now that it is, it makes sense to incorporate it into Tendermint2. diff --git a/docs/concepts/testnets.md b/docs/concepts/testnets.md index 7f0734cdc28..dd6d65085cc 100644 --- a/docs/concepts/testnets.md +++ b/docs/concepts/testnets.md @@ -6,7 +6,7 @@ id: testnets This page documents all Gno.land testnets, what their properties are, and how they are meant to be used. For testnet configuration, visit the -[reference section](../reference/network-config). +[reference section](../reference/network-config.md). Gno.land testnets are categorized by 4 main points: - **Persistence of state** diff --git a/docs/how-to-guides/simple-library.md b/docs/how-to-guides/simple-library.md index 1ae231251d0..923fc98922e 100644 --- a/docs/how-to-guides/simple-library.md +++ b/docs/how-to-guides/simple-library.md @@ -34,7 +34,7 @@ and [Visual Studio Code](https://marketplace.visualstudio.com/items?itemName=har ::: We discussed Gno folder structures more in detail in -the [simple Smart Contract guide](simple-contract.md#1-setting-up-the-work-directory). +the [simple Smart Contract guide](simple-contract.md#local-setup). For now, we will just follow some rules outlined there. Create the main working directory for our Package: diff --git a/docs/reference/network-config.md b/docs/reference/network-config.md index c2ec5409fc9..0da179e8f17 100644 --- a/docs/reference/network-config.md +++ b/docs/reference/network-config.md @@ -4,12 +4,12 @@ id: network-config # Network configurations -| Network | RPC Endpoint | Chain ID | -|-------------|------------------------------------|---------------| -| Portal Loop | https://rpc.gno.land:443 | `portal-loop` | -| Testnet 4 | upcoming | upcoming | -| Testnet 3 | https://rpc.test3.gno.land:443 | `test3` | -| Staging | https://rpc.staging.gno.land:26657 | `test3` | +| Network | RPC Endpoint | Chain ID | +|-------------|-----------------------------------|---------------| +| Portal Loop | https://rpc.gno.land:443 | `portal-loop` | +| Testnet 4 | upcoming | upcoming | +| Testnet 3 | https://rpc.test3.gno.land:443 | `test3` | +| Staging | http://rpc.staging.gno.land:26657 | `staging` | ### WebSocket endpoints All networks follow the same pattern for websocket connections: diff --git a/misc/docs-linter/go.mod b/misc/docs-linter/go.mod new file mode 100644 index 00000000000..2c2840e7a6d --- /dev/null +++ b/misc/docs-linter/go.mod @@ -0,0 +1,19 @@ +module linter + +go 1.21.6 + +require ( + github.com/gnolang/gno v0.0.0-20240516161351-0c9849a8ef0c + github.com/stretchr/testify v1.9.0 + golang.org/x/sync v0.7.0 + mvdan.cc/xurls/v2 v2.5.0 +) + +require ( + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/peterbourgon/ff/v3 v3.4.0 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + golang.org/x/sys v0.18.0 // indirect + golang.org/x/term v0.18.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) diff --git a/misc/docs-linter/go.sum b/misc/docs-linter/go.sum new file mode 100644 index 00000000000..ab8c3cf7c48 --- /dev/null +++ b/misc/docs-linter/go.sum @@ -0,0 +1,22 @@ +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/gnolang/gno v0.0.0-20240516161351-0c9849a8ef0c h1:jtZ+oN8ZpBM0wYbcFH0B7NjFFzTFqZZmZellSSKtaCE= +github.com/gnolang/gno v0.0.0-20240516161351-0c9849a8ef0c/go.mod h1:YcZbtNIfXVn4jS1pSG8SeG5RVHjyI7FPS3GypZaXxCI= +github.com/peterbourgon/ff/v3 v3.4.0 h1:QBvM/rizZM1cB0p0lGMdmR7HxZeI/ZrBWB4DqLkMUBc= +github.com/peterbourgon/ff/v3 v3.4.0/go.mod h1:zjJVUhx+twciwfDl0zBcFzl4dW8axCRyXE/eKY9RztQ= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= +golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= +golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/term v0.18.0 h1:FcHjZXDMxI8mM3nwhX9HlKop4C0YQvCVCdwYl2wOtE8= +golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +mvdan.cc/xurls/v2 v2.5.0 h1:lyBNOm8Wo71UknhUs4QTFUNNMyxy2JEIaKKo0RWOh+8= +mvdan.cc/xurls/v2 v2.5.0/go.mod h1:yQgaGQ1rFtJUzkmKiHYSSfuQxqfYmd//X6PxvholpeE= diff --git a/misc/docs-linter/main.go b/misc/docs-linter/main.go new file mode 100644 index 00000000000..029b2bf387a --- /dev/null +++ b/misc/docs-linter/main.go @@ -0,0 +1,218 @@ +package main + +import ( + "bufio" + "context" + "errors" + "flag" + "fmt" + "github.com/gnolang/gno/tm2/pkg/commands" + "golang.org/x/sync/errgroup" + "io" + "mvdan.cc/xurls/v2" + "net/http" + "os" + "path/filepath" + "strings" + "sync" +) + +var ( + errEmptyPath = errors.New("you need to pass in a path to scan") + err404Link = errors.New("link returned a 404") + errFound404Links = errors.New("found links resulting in a 404 response status") +) + +type cfg struct { + docsPath string +} + +func main() { + cfg := &cfg{} + + cmd := commands.NewCommand( + commands.Metadata{ + Name: "docs-linter", + ShortUsage: "docs-linter [flags]", + ShortHelp: "Finds broken 404 links in the .md files in the given folder & subfolders", + }, + cfg, + func(ctx context.Context, args []string) error { + return execLint(cfg, ctx) + }) + + cmd.Execute(context.Background(), os.Args[1:]) +} + +func (c *cfg) RegisterFlags(fs *flag.FlagSet) { + fs.StringVar( + &c.docsPath, + "path", + "./", + "path to dir to walk for .md files", + ) +} + +func execLint(cfg *cfg, ctx context.Context) error { + if cfg.docsPath == "" { + return errEmptyPath + } + + fmt.Println("Linting docs/") + + mdFiles, err := findFilePaths(cfg.docsPath) + if err != nil { + return fmt.Errorf("error finding .md files: %w", err) + } + + urlFileMap := make(map[string]string) + for _, filePath := range mdFiles { + // Extract URLs from each file + urls, err := extractUrls(filePath) + if err != nil { + fmt.Printf("Error extracting URLs from file: %s, %v", filePath, err) + continue + } + // For each url, save what file it was found in + for url, file := range urls { + urlFileMap[url] = file + } + } + + // Filter links by prefix & ignore localhost + var validUrls []string + for url := range urlFileMap { + // Look for http & https only + if strings.HasPrefix(url, "http://") || strings.HasPrefix(url, "https://") { + // Ignore localhost + if !strings.Contains(url, "localhost") && !strings.Contains(url, "127.0.0.1") { + validUrls = append(validUrls, url) + } + } + } + + // Setup parallel checking for links + g, _ := errgroup.WithContext(ctx) + + var ( + lock sync.Mutex + notFoundUrls []string + ) + + for _, url := range validUrls { + url := url + g.Go(func() error { + if err := checkUrl(url); err != nil { + lock.Lock() + notFoundUrls = append(notFoundUrls, fmt.Sprintf(">>> %s (found in file: %s)", url, urlFileMap[url])) + lock.Unlock() + } + + return nil + }) + } + + if err := g.Wait(); err != nil { + return err + } + + // Print out the URLs that returned a 404 along with the file names + if len(notFoundUrls) > 0 { + for _, result := range notFoundUrls { + fmt.Println(result) + } + + return errFound404Links + } + + return nil +} + +// findFilePaths gathers the file paths for specific file types +func findFilePaths(startPath string) ([]string, error) { + filePaths := make([]string, 0) + + walkFn := func(path string, info os.FileInfo, err error) error { + if err != nil { + return fmt.Errorf("error accessing file: %w", err) + } + + // Check if the file is a dir + if info.IsDir() { + return nil + } + + // Check if the file type matches + if !strings.HasSuffix(info.Name(), ".md") { + return nil + } + + // File is not a directory + filePaths = append(filePaths, path) + + return nil + } + + // Walk the directory root recursively + if walkErr := filepath.Walk(startPath, walkFn); walkErr != nil { + return nil, fmt.Errorf("unable to walk directory, %w", walkErr) + } + + return filePaths, nil +} + +// extractUrls extracts URLs from a file and maps them to the file +func extractUrls(filePath string) (map[string]string, error) { + file, err := os.Open(filePath) + if err != nil { + return nil, err + } + + cleanup := func() error { + if closeErr := file.Close(); closeErr != nil { + return fmt.Errorf("unable to gracefully close file, %w", closeErr) + } + return nil + } + + scanner := bufio.NewScanner(file) + urls := make(map[string]string) + + // Scan file line by line + for scanner.Scan() { + line := scanner.Text() + + // Extract links + rxStrict := xurls.Strict() + url := rxStrict.FindString(line) + + // Check for empty links and skip them + if url == " " || len(url) == 0 { + continue + } + + urls[url] = filePath + } + + return urls, cleanup() +} + +// checkUrl checks if a URL is a 404 +func checkUrl(url string) error { + // Attempt to retrieve the HTTP header + resp, err := http.Get(url) + if err != nil || resp.StatusCode == http.StatusNotFound { + return err404Link + } + + // Ensure the response body is closed properly + cleanup := func(Body io.ReadCloser) error { + if err := Body.Close(); err != nil { + return fmt.Errorf("could not close response properly: %w", err) + } + + return nil + } + + return cleanup(resp.Body) +} diff --git a/misc/docs-linter/main_test.go b/misc/docs-linter/main_test.go new file mode 100644 index 00000000000..ceb87fefeda --- /dev/null +++ b/misc/docs-linter/main_test.go @@ -0,0 +1,126 @@ +package main + +import ( + "context" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "os" + "path/filepath" + "sort" + "strconv" + "testing" + "time" +) + +func TestEmptyPathError(t *testing.T) { + t.Parallel() + + cfg := &cfg{ + docsPath: "", + } + + ctx, cancelFn := context.WithTimeout(context.Background(), time.Second*5) + defer cancelFn() + + assert.ErrorIs(t, execLint(cfg, ctx), errEmptyPath) +} + +func TestExtractLinks(t *testing.T) { + t.Parallel() + + // Generate temporary source dir + sourceDir, err := os.MkdirTemp(".", "sourceDir") + require.NoError(t, err) + t.Cleanup(removeDir(t, sourceDir)) + + // Create mock files with random links + mockFiles := map[string]string{ + "file1.md": "This is a test file with a link: https://example.com.\nAnother link: http://example.org.", + "file2.md": "Markdown content with a link: https://example.com/page.", + "file3.md": "Links in a list:\n- https://example.com/item1\n- https://example.org/item2", + } + + for fileName, content := range mockFiles { + filePath := filepath.Join(sourceDir, fileName) + err := os.WriteFile(filePath, []byte(content), 0644) + require.NoError(t, err) + } + + // Expected URLs and their corresponding files + expectedUrls := map[string]string{ + "https://example.com": filepath.Join(sourceDir, "file1.md"), + "http://example.org": filepath.Join(sourceDir, "file1.md"), + "https://example.com/page": filepath.Join(sourceDir, "file2.md"), + "https://example.com/item1": filepath.Join(sourceDir, "file3.md"), + "https://example.org/item2": filepath.Join(sourceDir, "file3.md"), + } + + // Extract URLs from each file in the sourceDir + for fileName := range mockFiles { + filePath := filepath.Join(sourceDir, fileName) + extractedUrls, err := extractUrls(filePath) + require.NoError(t, err) + + // Verify that the extracted URLs match the expected URLs + for url, expectedFile := range expectedUrls { + if expectedFile == filePath { + require.Equal(t, expectedFile, extractedUrls[url], "URL: %s not correctly mapped to file: %s", url, expectedFile) + } + } + } +} + +func TestFindFilePaths(t *testing.T) { + t.Parallel() + + tempDir, err := os.MkdirTemp(".", "test") + require.NoError(t, err) + t.Cleanup(removeDir(t, tempDir)) + + numSourceFiles := 20 + testFiles := make([]string, numSourceFiles) + + for i := 0; i < numSourceFiles; i++ { + testFiles[i] = "sourceFile" + strconv.Itoa(i) + ".md" + } + + for _, file := range testFiles { + filePath := filepath.Join(tempDir, file) + err := os.MkdirAll(filepath.Dir(filePath), os.ModePerm) + require.NoError(t, err) + + _, err = os.Create(filePath) + require.NoError(t, err) + } + + results, err := findFilePaths(tempDir) + require.NoError(t, err) + + expectedResults := make([]string, 0, len(testFiles)) + + for _, testFile := range testFiles { + expectedResults = append(expectedResults, filepath.Join(tempDir, testFile)) + } + + sort.Slice(results, func(i, j int) bool { + return results[i] < results[j] + }) + + sort.Slice(expectedResults, func(i, j int) bool { + return expectedResults[i] < expectedResults[j] + }) + + require.Equal(t, len(results), len(expectedResults)) + + for i, result := range results { + if result != expectedResults[i] { + require.Equal(t, result, expectedResults[i]) + } + } +} + +func removeDir(t *testing.T, dirPath string) func() { + return func() { + require.NoError(t, os.RemoveAll(dirPath)) + } +}