feat: add blank lines and comments (#73)

README.md

@@ -74,13 +74,25 @@ export default {
 If you need to use function in any rule, you need to create a config file through the `configPath` option
 
 ```js
-export default {
-  UserAgent: '*',
-  Disallow: '/',
+export default [
+  { UserAgent: '*' },
+  { Disallow: '/' },
+  { BlankLine: true },
+  { Comment: 'Comment here' },
 
   // Be aware that this will NOT work on target: 'static' mode
-  Sitemap: (req) => `https://${req.headers.host}/sitemap.xml`
-}
+  { Sitemap: (req) => `https://${req.headers.host}/sitemap.xml` }
+]
+```
+
+output: 
+
+```txt
+User-agent: *
+Disallow: /
+
+# Comment here
+Sitemap: https://robots.nuxtjs.org/sitemap.xml
 ```
 
 ### The keys and values available:
@@ -92,6 +104,8 @@ export default {
 - Host = `Host`
 - Sitemap = `Sitemap`
 - CleanParam = `Clean-param`
+- Comment = `# Comment`
+- BlankLine = `Add blank line`
 
 **Note:** Don't worry, keys are parsed with case insensitivity and special characters.
 

src/runtime/server/middleware.ts

@@ -1,25 +1,55 @@
-import { defineEventHandler, setHeader } from 'h3'
-import { RuleInterface, RuleValue, Rule } from '../../types'
+import { defineEventHandler, NodeIncomingMessage, setHeader } from 'h3'
+import { RuleValue, Rule } from '../../types'
 import robots from '#robots'
 
 export default defineEventHandler(async (event) => {
   setHeader(event, 'Content-Type', 'text/plain')
   return render(await getRules(robots, event.req))
 })
 
+enum Correspondence {
+  'User-agent',
+  'Crawl-delay',
+  'Disallow',
+  'Allow',
+  'Host',
+  'Sitemap',
+  'Clean-param',
+  'Comment',
+  'BlankLine'
+}
+
+interface RuleInterface {
+  key: Correspondence
+  value: string
+}
+
 function render (rules: RuleInterface[]) {
-  return rules.map(rule => `${rule.key}: ${String(rule.value).trim()}`).join('\n')
+  return rules.map((rule) => {
+    const value = String(rule.value).trim()
+
+    switch (rule.key.toString()) {
+      case Correspondence[Correspondence.Comment]:
+        return `# ${value}`
+      case Correspondence[Correspondence.BlankLine]:
+        return ''
+      default:
+        return `${rule.key}: ${value}`
+    }
+  }).join('\n')
 }
 
-async function getRules (options: Rule | Rule[], req = null) {
+async function getRules (options: Rule | Rule[], req: NodeIncomingMessage) {
   const correspondences = {
     useragent: 'User-agent',
     crawldelay: 'Crawl-delay',
     disallow: 'Disallow',
     allow: 'Allow',
     host: 'Host',
     sitemap: 'Sitemap',
-    cleanparam: 'Clean-param'
+    cleanparam: 'Clean-param',
+    comment: 'Comment',
+    blankline: 'BlankLine'
   }
 
   const rules: RuleInterface[] = []

src/types.ts

@@ -1,18 +1,3 @@
-export enum Correspondence {
-  'User-agent',
-  'Crawl-delay',
-  'Disallow',
-  'Allow',
-  'Host',
-  'Sitemap',
-  'Clean-param'
-}
-
-export interface RuleInterface {
-  key: Correspondence
-  value: string
-}
-
 export type RuleValue = string | boolean | Function | (string | boolean | Function)[]
 
 export type Rule = {

test/config-file.test.ts

@@ -8,6 +8,6 @@ describe('config file', async () => {
 
   test('render', async () => {
     const body = await $fetch('/robots.txt')
-    expect(body).toBe('User-agent: Googlebot\nUser-agent: Bingbot\nDisallow: /admin')
+    expect(body).toBe('User-agent: Googlebot\nUser-agent: Bingbot\n# Comment here\n\nDisallow: /admin')
   })
 })

test/fixture/config-file/robots.config.ts

@@ -1,4 +1,6 @@
-export default {
-  UserAgent: () => ['Googlebot', () => 'Bingbot'],
-  Disallow: '/admin'
-}
+export default [
+  { UserAgent: () => ['Googlebot', () => 'Bingbot'] },
+  { Comment: 'Comment here' },
+  { BlankLine: true },
+  { Disallow: '/admin' }
+]