docs: API auto-mentioning in docs (#910) (#1305)

* docs: api markdown code linker

* docs: api and command markdown code tests

* docs: manual links to api method pages removed

* docs: codeclimate fixes

* docs: test fixes

* docs-910: helpers refactoring

* docs-910: bringing back formatting

* docs-910: indent fix and redundant url removal

.gitignore

@@ -58,6 +58,7 @@ typings/
 *.log
 .idea
 .vscode
+.history
 
 # Mac finder artifacts
 .DS_Store

content/docs/api-reference/read.md

@@ -24,9 +24,9 @@ modelpkl = dvc.api.read(
 
 ## Description
 
-This function wraps [`dvc.api.open()`](/doc/api-reference/open), for a simple
-way to return the complete contents of a file tracked in a <abbr>DVC
-project</abbr>. The file can be tracked by DVC or by Git.
+This function wraps `dvc.api.open()`, for a simple way to return the complete
+contents of a file tracked in a <abbr>DVC project</abbr>. The file can be
+tracked by DVC or by Git.
 
 > This is similar to the `dvc get` command in our CLI.
 

content/docs/command-reference/list.md

@@ -21,7 +21,7 @@ DVC, by effectively replacing data files, models, directories with DVC-files
 files when you browse a <abbr>DVC repository</abbr> on Git hosting (e.g.
 Github), you just see the DVC-files. This makes it hard to navigate the project
 to find <abbr>data artifacts</abbr> for use with `dvc get`, `dvc import`, or
-[`dvc.api`](/doc/api-reference).
+`dvc.api`.
 
 `dvc list` prints a virtual view of a DVC repository, as if files and
 directories [tracked by DVC](/doc/use-cases/versioning-data-and-model-files)
@@ -48,7 +48,7 @@ list files recursively.
 
 Please note that `dvc list` doesn't check whether the listed data (tracked by
 DVC) actually exists in remote storage, so it's not guaranteed whether it can be
-accessed with `dvc get`, `dvc import`, or [`dvc.api`](/doc/api-reference)
+accessed with `dvc get`, `dvc import`, or `dvc.api`
 
 ## Options
 

content/docs/install/index.md

@@ -12,7 +12,7 @@
 DVC can be used as a Python library, simply install it with a package manager
 like `pip` or `conda`, and as a Python
 [project requirement](https://pip.pypa.io/en/latest/user_guide/#requirements-files)
-if needed. The [Python API](/doc/api-reference) module is `dvc.api`.
+if needed. The Python API module is `dvc.api`.
 
 ## Advanced options
 

content/docs/use-cases/data-registries.md

@@ -88,8 +88,8 @@ $ dvc push
 ## Using registries
 
 The main methods to consume <abbr>data artifacts</abbr> from a **data registry**
-are the `dvc import` and `dvc get` commands, as well as the
-[`dvc.api`](/doc/api-reference) Python API.
+are the `dvc import` and `dvc get` commands, as well as the `dvc.api` Python
+API.
 
 But first, you may want to explore the contents of a data DVC repo.
 
@@ -164,9 +164,9 @@ the project dependency metadata in the import stage (DVC-file).
 
 ### Programmatic reusability of DVC data
 
-Our Python API, included with the `dvc` package installed with DVC, includes the
-`open` function to load/stream data directly from external <abbr>DVC
-projects</abbr>:
+Our Python API (`dvc.api`), included with the `dvc` package installed with DVC,
+includes the `open` function to load/stream data directly from external
+<abbr>DVC projects</abbr>:
 
 ```python
 import dvc.api.open
@@ -182,7 +182,7 @@ with dvc.api.open(model_path, repo_url) as fd:
 This opens `model.pkl` as a file descriptor. The example above tries to
 illustrate a hardcoded ML model **deployment** method.
 
-> Notice that the `dvc.api.get_url` and `dvc.api.read` functions are also
+> Notice that the `dvc.api.get_url()` and `dvc.api.read()` functions are also
 > available.
 
 ## Updating registries

content/docs/user-guide/contributing/docs.md

@@ -155,6 +155,10 @@ is installed when `yarn` runs (explained above).
 - Markdown: Using `dvc <command>`, the docs engine will create a link to that
   command automatically. (No need to use `[]()` explicitly to create them.)
 
+- Markdown: Using `dvc.api.<api_method>()` or `dvc.api`, the docs engine will
+  create a link to that API method automatically. (No need to use `[]()`
+  explicitly to create them.)
+
 - Markdown: Neither bullet lists nor each item's should be too long (3 sentence
   paragraphs max.) Full sentence bullets should begin with a capital letter and
   end in period `.` otherwise they can be all lower case and have no ending

package.json

@@ -100,7 +100,7 @@
     "@typescript-eslint/parser": "^2.27.0",
     "autoprefixer": "^9.7.6",
     "babel-eslint": "^10.1.0",
-    "babel-jest": "^25.2.6",
+    "babel-jest": "^26.0.1",
     "babel-plugin-transform-define": "^2.0.0",
     "babel-plugin-transform-object-assign": "^6.22.0",
     "eslint": "^6.8.0",
@@ -137,7 +137,7 @@
     "gatsby-transformer-sharp": "2.2.23",
     "hast-util-select": "^4.0.0",
     "husky": "^4.2.3",
-    "jest": "^25.2.7",
+    "jest": "^26.0.1",
     "lint-staged": "^10.1.2",
     "postcss-color-mod-function": "^3.0.3",
     "postcss-custom-media": "^7.0.8",
@@ -149,9 +149,11 @@
     "rehype-stringify": "^7.0.0",
     "remark": "^12.0.0",
     "remark-html": "^11.0.1",
+    "remark-parse": "^8.0.2",
     "stylelint": "^13.3.0",
     "stylelint-config-standard": "^20.0.0",
-    "typescript": "^3.8.3"
+    "typescript": "^3.8.3",
+    "unist-util-remove-position": "^2.0.1"
   },
   "husky": {
     "hooks": {

plugins/gatsby-remark-dvc-linker/apiLinker.js

@@ -0,0 +1,35 @@
+/* eslint-env node */
+
+const { createLinkNode } = require('./helpers')
+const { getItemByPath } = require('../../src/utils/shared/sidebar')
+
+const DVC_API_REGEXP = /dvc.api([a-z-._]*\(\)$)?/
+const METHOD_REGEXP = /^[a-z-._]*\(\)$/
+const API_ROOT = '/doc/api-reference/'
+
+module.exports = astNode => {
+  const node = astNode[0]
+  const parent = astNode[2]
+
+  if (parent.type !== 'link' && DVC_API_REGEXP.test(node.value)) {
+    const parts = node.value.split('.')
+    let url
+
+    const isMethod = parts[2] && METHOD_REGEXP.test(parts[2])
+    const method = isMethod && parts[2].slice(0, -2)
+    const isRoot = parts[0] === 'dvc' && parts[1] === 'api' && !parts[2]
+
+    if (isRoot) {
+      url = `${API_ROOT}`
+    } else {
+      url = `${API_ROOT}${method}`
+    }
+
+    const isMethodPageExists = getItemByPath(url)
+    if (isMethodPageExists) {
+      createLinkNode(url, astNode)
+    }
+  }
+
+  return astNode
+}

plugins/gatsby-remark-dvc-linker/commandLinker.js

@@ -0,0 +1,37 @@
+/* eslint-env node */
+
+const { createLinkNode } = require('./helpers')
+const { getItemByPath } = require('../../src/utils/shared/sidebar')
+
+const DVC_REGEXP = /dvc\s+[a-z][a-z-.]*/
+const COMMAND_REGEXP = /^[a-z][a-z-]*$/
+const COMMAND_ROOT = '/doc/command-reference/'
+
+module.exports = astNode => {
+  const node = astNode[0]
+  const parent = astNode[2]
+
+  if (parent.type !== 'link' && DVC_REGEXP.test(node.value)) {
+    const parts = node.value.split(/\s+/)
+    let url
+
+    const hasThirdSegment = parts[2] && COMMAND_REGEXP.test(parts[2])
+    const isCommandPageExists = getItemByPath(`${COMMAND_ROOT}${parts[1]}`)
+    const isSubcommandPageExists =
+      isCommandPageExists &&
+      hasThirdSegment &&
+      getItemByPath(`${COMMAND_ROOT}${parts[1]}/${parts[2]}`)
+
+    if (isSubcommandPageExists) {
+      url = `${COMMAND_ROOT}${parts[1]}/${parts[2]}`
+    } else if (isCommandPageExists && hasThirdSegment) {
+      url = `${COMMAND_ROOT}${parts[1]}#${parts[2]}`
+    } else if (isCommandPageExists) {
+      url = `${COMMAND_ROOT}${parts[1]}`
+    }
+
+    createLinkNode(url, astNode)
+  }
+
+  return astNode
+}

plugins/gatsby-remark-dvc-linker/helpers.js

@@ -0,0 +1,20 @@
+const unified = require('unified')
+const remarkHtml = require('remark-html')
+const remarkParse = require('remark-parse')
+const removePosition = require('unist-util-remove-position')
+
+// We do not need to consider the position of the AST nodes
+const buildAst = mdToBuild =>
+  removePosition(unified().use(remarkHtml).use(remarkParse).parse(mdToBuild))
+
+const createLinkNode = (url, [node, index, parent]) =>
+  url &&
+  (parent.children[index] = {
+    type: 'link',
+    url,
+    title: null,
+    children: [node],
+    position: node.position
+  })
+
+module.exports = { buildAst, createLinkNode }

plugins/gatsby-remark-dvc-linker/index.js

@@ -1,43 +1,19 @@
 /* eslint-env node */
 
+const flow = require('lodash/flow')
+const constant = require('lodash/constant')
 const visit = require('unist-util-visit')
-const { getItemByPath } = require('../../src/utils/shared/sidebar')
 
-const DVC_REGEXP = /dvc\s+[a-z][a-z-.]*/
-const COMMAND_REGEXP = /^[a-z][a-z-]*$/
-const COMMAND_ROOT = '/doc/command-reference/'
+const apiLinker = require('./apiLinker')
+const commandLinker = require('./commandLinker')
 
+// Lifting up the AST visitor in order not to repeat the
+// calculations times the amount of linkers we have
 module.exports = ({ markdownAST }) => {
-  visit(markdownAST, 'inlineCode', function (node, index, parent) {
-    if (parent.type !== 'link' && DVC_REGEXP.test(node.value)) {
-      const parts = node.value.split(/\s+/)
-      let url
-
-      const hasThirdSegment = parts[2] && COMMAND_REGEXP.test(parts[2])
-      const isCommandPageExists = getItemByPath(`${COMMAND_ROOT}${parts[1]}`)
-      const isSubcommandPageExists =
-        isCommandPageExists &&
-        hasThirdSegment &&
-        getItemByPath(`${COMMAND_ROOT}${parts[1]}/${parts[2]}`)
-
-      if (isSubcommandPageExists) {
-        url = `${COMMAND_ROOT}${parts[1]}/${parts[2]}`
-      } else if (isCommandPageExists && hasThirdSegment) {
-        url = `${COMMAND_ROOT}${parts[1]}#${parts[2]}`
-      } else if (isCommandPageExists) {
-        url = `${COMMAND_ROOT}${parts[1]}`
-      }
-
-      if (url) {
-        parent.children[index] = {
-          type: 'link',
-          url: url,
-          children: [node],
-          position: node.position
-        }
-      }
-    }
-  })
-
+  visit(
+    markdownAST,
+    'inlineCode',
+    flow([Array, commandLinker, apiLinker, constant(undefined)])
+  )
   return markdownAST
 }

plugins/gatsby-remark-dvc-linker/index.test.js

@@ -0,0 +1,58 @@
+const flow = require('lodash/flow')
+const constant = require('lodash/constant')
+const visit = require('unist-util-visit')
+
+const gatsbyRemarkDvcLinker = require('.')
+const apiLinker = require('./apiLinker')
+const commandLinker = require('./commandLinker')
+
+const { buildAst } = require('./helpers')
+
+describe('gatsby-remark-dvc-linker', () => {
+  api = {
+    inlineCode: '`dvc.api.get_url()`',
+    url: '[`dvc.api.get_url()`](/doc/api-reference/get_url)'
+  }
+
+  apiRoot = {
+    inlineCode: '`dvc.api`',
+    url: '[`dvc.api`](/doc/api-reference/)'
+  }
+
+  command = {
+    inlineCode: '`dvc get`',
+    url: '[`dvc get`](/doc/command-reference/get)'
+  }
+
+  it('composes apiLinker and commandLinker', () => {
+    const ast = buildAst(`${api.inlineCode} ${command.inlineCode}`)
+    gatsbyRemarkDvcLinker({ markdownAST: ast })
+    expect(ast).toEqual(buildAst(`${api.url} ${command.url}`))
+  })
+
+  describe('apiLinker', () => {
+    it('transforms API reference to a link', () => {
+      const ast = buildAst(api.inlineCode)
+      visit(ast, 'inlineCode', flow([Array, apiLinker, constant(undefined)]))
+      expect(ast).toEqual(buildAst(api.url))
+    })
+
+    it('transforms root API reference to a link', () => {
+      const ast = buildAst(apiRoot.inlineCode)
+      visit(ast, 'inlineCode', flow([Array, apiLinker, constant(undefined)]))
+      expect(ast).toEqual(buildAst(apiRoot.url))
+    })
+  })
+
+  describe('commandLinker', () => {
+    it('transforms command reference to a link', () => {
+      const ast = buildAst(command.inlineCode)
+      visit(
+        ast,
+        'inlineCode',
+        flow([Array, commandLinker, constant(undefined)])
+      )
+      expect(ast).toEqual(buildAst(command.url))
+    })
+  })
+})