feat(docs): adds flag to delete docs from ReadMe if they're no longer…

… in local folder (#581)

* feat(openapi): Add `updateSingleSpec` option to automatically update an only available spec file without any prompts

* feat(docs): Updated docs

* Fix eslint errors

* Throw error if using --updateSingleSpec when there are multiple spec files available

* Show a warning when passing both `--updateSingleSpec` and `--id`

* Update tests snapshot

* docs: copy edits, rename flag to `--update`

* fix: add logic for if create and update flags passed together

* ci: attempt to skip failing step if API key isn't present

* Revert "ci: attempt to skip failing step if API key isn't present"

This reverts commit 95a8148.

* Add --deleteMissing option to `rdme docs` command to delete from ReadMe documents that don't exist on the local folder

* Take into account only "guide" categories

* chore: markdown formatting

* Add warning about what the new option does

* Improved typing of functions

* Move "if" to a more proper location

* Refactor "getSlug" to "readDoc" and reuse code

* Removed unused code

* Simplify code

* Refactored tests to use a folder with a document that should not be deleted

* Added a confirmation prompt when folder is empty

* Rename --deleteMissing to --cleanup

* CR

* Undo accidental commit

* Update index.ts

Co-authored-by: Kanad Gupta <[email protected]>

README.md

@@ -204,6 +204,10 @@ rdme docs path-to-markdown-files --version={project-version}
 
 This command also has a dry run mode, which can be useful for initial setup and debugging. You can read more about dry run mode [in our docs](https://docs.readme.com/docs/rdme#dry-run-mode).
 
+#### Cleanup
+
+If you wish to delete documents from ReadMe that are no longer present in your local directory, pass the `--cleanup` option to the command.
+
 #### Edit a Single ReadMe Doc on Your Local Machine
 
 ```sh

__tests__/__fixtures__/docs/delete-docs/some-doc.md

@@ -0,0 +1,4 @@
+---
+category: 5ae122e10fdf4e39bb34db6f
+title: This is the document title
+---

__tests__/cmds/docs/index.test.ts

@@ -1,4 +1,5 @@
 /* eslint-disable no-console */
+import crypto from 'crypto';
 import fs from 'fs';
 import path from 'path';
 
@@ -380,6 +381,124 @@ describe('rdme docs', () => {
     });
   });
 
+  describe('cleanup docs', () => {
+    const folder = `./__tests__/${fixturesBaseDir}/delete-docs`;
+    const someDocContent = fs.readFileSync(path.join(folder, 'some-doc.md'));
+    const lastUpdatedHash = crypto.createHash('sha1').update(someDocContent).digest('hex');
+    let consoleWarnSpy;
+
+    function getWarningCommandOutput() {
+      return [consoleWarnSpy.mock.calls.join('\n\n')].filter(Boolean).join('\n\n');
+    }
+
+    beforeEach(() => {
+      consoleWarnSpy = jest.spyOn(console, 'warn').mockImplementation();
+    });
+
+    afterEach(() => {
+      consoleWarnSpy.mockRestore();
+    });
+
+    it('should delete doc if file is missing and --cleanup option is used', async () => {
+      const versionMock = getAPIMock()
+        .get(`/api/v1/version/${version}`)
+        .basicAuth({ user: key })
+        .reply(200, { version });
+
+      const apiMocks = getAPIMockWithVersionHeader(version)
+        .get('/api/v1/categories?perPage=20&page=1')
+        .basicAuth({ user: key })
+        .reply(200, [{ slug: 'category1', type: 'guide' }], { 'x-total-count': '1' })
+        .get('/api/v1/categories/category1/docs')
+        .basicAuth({ user: key })
+        .reply(200, [{ slug: 'this-doc-should-be-missing-in-folder' }, { slug: 'some-doc' }])
+        .delete('/api/v1/docs/this-doc-should-be-missing-in-folder')
+        .basicAuth({ user: key })
+        .reply(204, '')
+        .get('/api/v1/docs/some-doc')
+        .basicAuth({ user: key })
+        .reply(200, { lastUpdatedHash });
+
+      await expect(
+        docs.run({
+          folder,
+          key,
+          version,
+          cleanup: true,
+        })
+      ).resolves.toBe(
+        '🗑️ successfully deleted `this-doc-should-be-missing-in-folder`.\n' +
+          '`some-doc` was not updated because there were no changes.'
+      );
+      const warningOutput = getWarningCommandOutput();
+      expect(warningOutput).toBe(
+        "⚠️  Warning! We're going to delete from ReadMe any document that isn't found in ./__tests__/__fixtures__/docs/delete-docs."
+      );
+
+      apiMocks.done();
+      versionMock.done();
+    });
+
+    it('should return doc delete info for dry run', async () => {
+      const versionMock = getAPIMock()
+        .get(`/api/v1/version/${version}`)
+        .basicAuth({ user: key })
+        .reply(200, { version });
+      const apiMocks = getAPIMockWithVersionHeader(version)
+        .get('/api/v1/categories?perPage=20&page=1')
+        .basicAuth({ user: key })
+        .reply(200, [{ slug: 'category1', type: 'guide' }], { 'x-total-count': '1' })
+        .get('/api/v1/categories/category1/docs')
+        .basicAuth({ user: key })
+        .reply(200, [{ slug: 'this-doc-should-be-missing-in-folder' }])
+        .get('/api/v1/docs/some-doc')
+        .basicAuth({ user: key })
+        .reply(200, { lastUpdatedHash });
+      await expect(
+        docs.run({
+          folder,
+          key,
+          version,
+          cleanup: true,
+          dryRun: true,
+        })
+      ).resolves.toBe(
+        '🎭 dry run! This will delete `this-doc-should-be-missing-in-folder`.\n' +
+          '🎭 dry run! `some-doc` will not be updated because there were no changes.'
+      );
+      const warningOutput = getWarningCommandOutput();
+      expect(warningOutput).toBe(
+        "⚠️  Warning! We're going to delete from ReadMe any document that isn't found in ./__tests__/__fixtures__/docs/delete-docs."
+      );
+
+      apiMocks.done();
+      versionMock.done();
+    });
+
+    it('should do nothing if using --cleanup but the folder is empty and the user aborted', async () => {
+      prompts.inject([false]);
+
+      const versionMock = getAPIMock()
+        .get(`/api/v1/version/${version}`)
+        .basicAuth({ user: key })
+        .reply(200, { version });
+
+      await expect(
+        docs.run({
+          folder: './__tests__/__fixtures__/ref-oas',
+          key,
+          version,
+          cleanup: true,
+        })
+      ).rejects.toStrictEqual(new Error('Aborting, no changes were made.'));
+
+      const warningOutput = getWarningCommandOutput();
+      expect(warningOutput).toBe('');
+
+      versionMock.done();
+    });
+  });
+
   describe('slug metadata', () => {
     it('should use provided slug', async () => {
       const slug = 'new-doc-slug';

src/cmds/docs/index.ts

@@ -5,15 +5,26 @@ import config from 'config';
 
 import Command, { CommandCategories } from '../../lib/baseCommand';
 import createGHA from '../../lib/createGHA';
+import deleteDoc from '../../lib/deleteDoc';
+import getDocs from '../../lib/getDocs';
+import * as promptHandler from '../../lib/prompts';
+import promptTerminal from '../../lib/promptWrapper';
 import pushDoc from '../../lib/pushDoc';
 import readdirRecursive from '../../lib/readdirRecursive';
+import readDoc from '../../lib/readDoc';
 import { getProjectVersion } from '../../lib/versionSelect';
 
 export type Options = {
   dryRun?: boolean;
   folder?: string;
+  cleanup?: boolean;
 };
 
+function getSlug(filename: string): string {
+  const { slug } = readDoc(filename);
+  return slug;
+}
+
 export default class DocsCommand extends Command {
   constructor() {
     super();
@@ -39,13 +50,18 @@ export default class DocsCommand extends Command {
         type: Boolean,
         description: 'Runs the command without creating/updating any docs in ReadMe. Useful for debugging.',
       },
+      {
+        name: 'cleanup',
+        type: Boolean,
+        description: 'Delete any docs from ReadMe if their slugs are not found in the target folder.',
+      },
     ];
   }
 
   async run(opts: CommandOptions<Options>) {
     super.run(opts);
 
-    const { dryRun, folder, key, version } = opts;
+    const { dryRun, folder, key, version, cleanup } = opts;
 
     if (!folder) {
       return Promise.reject(new Error(`No folder provided. Usage \`${config.get('cli')} ${this.usage}\`.`));
@@ -65,7 +81,25 @@ export default class DocsCommand extends Command {
 
     Command.debug(`number of files: ${files.length}`);
 
-    if (!files.length) {
+    const changes: string[] = [];
+    if (cleanup) {
+      if (!files.length) {
+        const { deleteAll } = await promptTerminal(promptHandler.deleteDocsPrompt(selectedVersion));
+        if (!deleteAll) {
+          return Promise.reject(new Error('Aborting, no changes were made.'));
+        }
+      }
+
+      Command.warn(`We're going to delete from ReadMe any document that isn't found in ${folder}.`);
+      const docs = await getDocs(key, selectedVersion);
+      const docSlugs = docs.map(({ slug }: { slug: string }) => slug);
+      const fileSlugs = new Set(files.map(getSlug));
+      const slugsToDelete = docSlugs.filter((slug: string) => !fileSlugs.has(slug));
+      const deletedDocs = await Promise.all(
+        slugsToDelete.map((slug: string) => deleteDoc(key, selectedVersion, dryRun, slug, this.cmdCategory))
+      );
+      changes.push(...deletedDocs);
+    } else if (!files.length) {
       return Promise.reject(new Error(`We were unable to locate Markdown files in ${folder}.`));
     }
 
@@ -74,8 +108,8 @@ export default class DocsCommand extends Command {
         return pushDoc(key, selectedVersion, dryRun, filename, this.cmdCategory);
       })
     );
-
-    return Promise.resolve(chalk.green(updatedDocs.join('\n'))).then(msg =>
+    changes.push(...updatedDocs);
+    return Promise.resolve(chalk.green(changes.join('\n'))).then(msg =>
       createGHA(msg, this.command, this.args, { ...opts, version: selectedVersion })
     );
   }

src/lib/deleteDoc.ts

@@ -0,0 +1,40 @@
+import type { CommandCategories } from './baseCommand';
+
+import config from 'config';
+import { Headers } from 'node-fetch';
+
+import fetch, { cleanHeaders, handleRes } from './fetch';
+
+/**
+ * Delete a document from ReadMe
+ *
+ * @param {String} key the project API key
+ * @param {String} selectedVersion the project version
+ * @param {Boolean} dryRun boolean indicating dry run mode
+ * @param {String} slug The slug of the document to delete
+ * @param {String} type module within ReadMe to update (e.g. docs, changelogs, etc.)
+ * @returns {Promise<String>} a string containing the result
+ */
+export default async function deleteDoc(
+  key: string,
+  selectedVersion: string,
+  dryRun: boolean,
+  slug: string,
+  type: CommandCategories
+): Promise<string> {
+  if (dryRun) {
+    return Promise.resolve(`🎭 dry run! This will delete \`${slug}\`.`);
+  }
+  return fetch(`${config.get('host')}/api/v1/${type}/${slug}`, {
+    method: 'delete',
+    headers: cleanHeaders(
+      key,
+      new Headers({
+        'x-readme-version': selectedVersion,
+        'Content-Type': 'application/json',
+      })
+    ),
+  })
+    .then(handleRes)
+    .then(() => `🗑️ successfully deleted \`${slug}\`.`);
+}

src/lib/fetch.ts

@@ -9,6 +9,8 @@ import APIError from './apiError';
 import { isGHA } from './isCI';
 import { debug } from './logger';
 
+const SUCCESS_NO_CONTENT = 204;
+
 /**
  * Getter function for a string to be used in the user-agent header based on the current
  * environment.
@@ -71,6 +73,9 @@ async function handleRes(res: Response) {
     }
     return body;
   }
+  if (res.status === SUCCESS_NO_CONTENT) {
+    return {};
+  }
   // If we receive a non-JSON response, it's likely an error.
   // Let's debug the raw response body and throw it.
   const body = await res.text();

src/lib/getDocs.ts

@@ -0,0 +1,46 @@
+import config from 'config';
+import { Headers } from 'node-fetch';
+
+import fetch, { cleanHeaders, handleRes } from './fetch';
+import getCategories from './getCategories';
+
+type Document = {
+  _id: string;
+  title: string;
+  slug: string;
+  order: number;
+  hidden: boolean;
+  children: Document[];
+};
+
+function flatten(data: Document[][]): Document[] {
+  return [].concat(...data);
+}
+
+async function getCategoryDocs(key: string, selectedVersion: string, category: string): Promise<Document[]> {
+  return fetch(`${config.get('host')}/api/v1/categories/${category}/docs`, {
+    method: 'get',
+    headers: cleanHeaders(
+      key,
+      new Headers({
+        'x-readme-version': selectedVersion,
+        'Content-Type': 'application/json',
+      })
+    ),
+  }).then(handleRes);
+}
+
+/**
+ * Retrieve the docs under all categories or under a specific one
+ *
+ * @param {String} key the project API key
+ * @param {String} selectedVersion the project version
+ * @returns {Promise<Array<Document>>} an array containing the docs
+ */
+export default async function getDocs(key: string, selectedVersion: string): Promise<Document[]> {
+  return getCategories(key, selectedVersion)
+    .then(categories => categories.filter(({ type }: { type: string }) => type === 'guide'))
+    .then(categories => categories.map(({ slug }: { slug: string }) => getCategoryDocs(key, selectedVersion, slug)))
+    .then(categoryDocsPromises => Promise.all(categoryDocsPromises))
+    .then(flatten);
+}

src/lib/prompts.ts

@@ -220,3 +220,11 @@ export function createVersionPrompt(
     },
   ];
 }
+
+export function deleteDocsPrompt(version: string): PromptObject {
+  return {
+    type: 'confirm',
+    name: 'deleteAll',
+    message: `This action will delete all docs under version ${version} from ReadMe, would you like to continue?`,
+  };
+}

src/lib/pushDoc.ts

@@ -1,16 +1,14 @@
 import crypto from 'crypto';
-import fs from 'fs';
-import path from 'path';
 
 import chalk from 'chalk';
 import config from 'config';
-import grayMatter from 'gray-matter';
 import { Headers } from 'node-fetch';
 
 import APIError from './apiError';
 import { CommandCategories } from './baseCommand';
 import fetch, { cleanHeaders, handleRes } from './fetch';
 import { debug } from './logger';
+import readDoc from './readDoc';
 
 /**
  * Reads the contents of the specified Markdown or HTML file
@@ -31,14 +29,8 @@ export default async function pushDoc(
   filepath: string,
   type: CommandCategories
 ) {
-  debug(`reading file ${filepath}`);
-  const file = fs.readFileSync(filepath, 'utf8');
-  const matter = grayMatter(file);
-  debug(`frontmatter for ${filepath}: ${JSON.stringify(matter)}`);
-
-  // Stripping the subdirectories and markdown extension from the filename and lowercasing to get the default slug.
-  const slug = matter.data.slug || path.basename(filepath).replace(path.extname(filepath), '').toLowerCase();
-  const hash = crypto.createHash('sha1').update(file).digest('hex');
+  const { content, matter, slug } = readDoc(filepath);
+  const hash = crypto.createHash('sha1').update(content).digest('hex');
 
   let data: {
     body?: string;