Delegate markdown rendering to the frontend

datahub/server/lib/change_log.py

@@ -1,38 +1,15 @@
 import os
 from itertools import islice
-import markdown2
 from const.path import CHANGE_LOG_PATH
 
 
 __change_logs = None
 
 
 def generate_change_log(raw_text: str) -> str:
-    # Since the markdown is used for the documentation site
-    # We need to preprocess it so that it can be presentable
-    # within DataHub
-
     # TODO: either move the changelog completely to documentation site
     #       or come up with a solution that can be compatible with both
-
-    lines = raw_text.split("\n")
-    filtered_lines = []
-
-    # Remove --- blocks from markdown
-    inside_comment = False
-    for line in lines:
-        if line.startswith("---"):
-            inside_comment = not inside_comment
-        if not inside_comment:
-            filtered_lines.append(line)
-
-    # Add "static" prefix to image path
-    filtered_lines = [
-        line.replace("![](/changelog/", "![](/static/changelog/")
-        for line in filtered_lines
-    ]
-
-    return markdown2.markdown("\n".join(filtered_lines))
+    return raw_text.replace("![](/changelog/", "![](/static/changelog/")
 
 
 def load_all_change_logs():
@@ -46,12 +23,7 @@ def load_all_change_logs():
             with open(os.path.join(CHANGE_LOG_PATH, "./{}".format(filename))) as f:
                 changelog_date = filename.split(".")[0]
                 __change_logs.append(
-                    {
-                        "date": changelog_date,
-                        "content": generate_change_log(
-                            f"{changelog_date}\n" + f.read()
-                        ),
-                    }
+                    {"date": changelog_date, "content": generate_change_log(f.read()),}
                 )
     return __change_logs
 

datahub/server/lib/query_executor/executor_template/templates.py

@@ -14,10 +14,7 @@
 jdbc:hive2://<host1>:<port1>,<host2>:<port2>/dbName;sess_var_list?hive_conf_list#hive_var_list
 </p>
 <p>Currently support zookeeper in session var, and will pass any conf variables to HS2</p>
-<p>See
-    <a href="https://cwiki.apache.org/confluence/display/Hive/HiveServer2+Clients#HiveServer2Clients-JDBC">
-        here
-    </a> for more details.
+<p>See [here](https://cwiki.apache.org/confluence/display/Hive/HiveServer2+Clients#HiveServer2Clients-JDBC) for more details.
 </p>""",
     ),
     username=FormField(regex="\\w+"),
@@ -32,7 +29,7 @@
         helper="""
 <p>Format jdbc:presto://&lt;host:port&gt;/&lt;catalog&gt;/&lt;schema&gt;?presto_conf_list</p>
 <p>Catalog and schema are optional. We only support SSL as the conf option.</p>
-<p>See <a href="https://prestodb.github.io/docs/current/installation/jdbc.html">here</a> for more details.</p>""",
+<p>See [here](https://prestodb.github.io/docs/current/installation/jdbc.html) for more details.</p>""",
     ),
     username=FormField(regex="\\w+"),
     password=FormField(hidden=True),
@@ -44,14 +41,13 @@
         required=True,
         helper="""
 <p>
-    See <a href="https://docs.sqlalchemy.org/en/latest/core/engines.html#database-urls">here</a>
-    for more details.
+    See [here](https://docs.sqlalchemy.org/en/latest/core/engines.html#database-urls) for more details.
 </p>""",
     )
 )
 
 bigquery_template = StructFormField(
     google_credentials_json=FormField(
-        helper="The JSON string used to log in as service account. If not provided then GOOGLE_CREDS from settings will be used.",
+        helper="The JSON string used to log in as service account. If not provided then **GOOGLE_CREDS** from settings will be used.",
     )
 )

datahub/server/models/admin.py

@@ -1,4 +1,3 @@
-import markdown2
 import sqlalchemy as sql
 from sqlalchemy.orm import relationship, backref
 
@@ -32,7 +31,7 @@ class Announcement(CRUDMixin, Base):
     def to_dict(self):
         return {
             "id": self.id,
-            "message": markdown2.markdown(self.message),
+            "message": self.message,
             "url_regex": self.url_regex,
             "can_dismiss": self.can_dismiss,
         }

datahub/webapp/__tests__/lib/markdown.test.ts

@@ -0,0 +1,32 @@
+import { sanitizeAndExtraMarkdown } from 'lib/markdown';
+
+describe('sanitizeAndExtraMarkdown', () => {
+    it('Works for standard markdown', () => {
+        const markdown = 'Hello **World** ---\nHello There.';
+        expect(sanitizeAndExtraMarkdown(markdown)).toEqual([markdown, {}]);
+    });
+
+    it('Extract Properties for --- markdown', () => {
+        const markdown = `---
+foo: bar
+bar: baz
+---
+Hello **World**`;
+        expect(sanitizeAndExtraMarkdown(markdown)).toEqual([
+            'Hello **World**',
+            { foo: 'bar', bar: 'baz' },
+        ]);
+    });
+
+    it('Only extracts valid properties', () => {
+        const markdown = `---
+foobar
+bar: baz
+---
+Hello **World**`;
+        expect(sanitizeAndExtraMarkdown(markdown)).toEqual([
+            'Hello **World**',
+            { bar: 'baz' },
+        ]);
+    });
+});

datahub/webapp/components/Announcements/Announcements.tsx

@@ -1,3 +1,4 @@
+import Markdown from 'markdown-to-jsx';
 import React from 'react';
 import { useDispatch } from 'react-redux';
 import * as datahubUIActions from 'redux/dataHubUI/action';
@@ -46,7 +47,7 @@ export const Announcements: React.FunctionComponent = () => {
                     }`}
                     key={item.id}
                 >
-                    <p dangerouslySetInnerHTML={{ __html: item.message }} />
+                    <Markdown>{item.message}</Markdown>
                     {deleteButton}
                 </Level>
             );

datahub/webapp/components/ChangeLog/ChangeLog.tsx

@@ -1,8 +1,10 @@
 import * as React from 'react';
 import { useParams } from 'react-router-dom';
+import Markdown from 'markdown-to-jsx';
 
 import ds from 'lib/datasource';
 import localStore from 'lib/local-store';
+import { sanitizeAndExtraMarkdown } from 'lib/markdown';
 import { navigateWithinEnv } from 'lib/utils/query-string';
 import { ChangeLogValue, CHANGE_LOG_KEY } from 'lib/local-store/const';
 
@@ -16,6 +18,20 @@ interface IChangeLogItem {
     date: string;
 }
 
+const ChangeLogMarkdown: React.FC<{ markdown: string }> = ({ markdown }) => {
+    const processedMarkdown = React.useMemo(() => {
+        const [text, properties] = sanitizeAndExtraMarkdown(markdown);
+        return 'title' in properties
+            ? `# ${properties['title']}\n` + text
+            : text;
+    }, [markdown]);
+    return (
+        <Content className="ChangeLog-content mt12 mh12 mb24">
+            <Markdown>{processedMarkdown}</Markdown>
+        </Content>
+    );
+};
+
 export const ChangeLog: React.FunctionComponent = () => {
     const { date: changeLogDate } = useParams();
     const [changeLogContent, setChangeLogContent] = React.useState<string[]>(
@@ -54,11 +70,7 @@ export const ChangeLog: React.FunctionComponent = () => {
     }, [changeLogDate]);
 
     const changeLogDOM = changeLogContent.map((text, idx) => (
-        <Content
-            className="ChangeLog-content mt12 mh12 mb24"
-            dangerouslySetInnerHTML={{ __html: text }}
-            key={idx}
-        />
+        <ChangeLogMarkdown markdown={text} key={idx} />
     ));
     const changeLogListDOM = changeLogDate ? null : (
         <div className="ChangeLog-list">

datahub/webapp/components/DataDocStatementExecution/ErrorSuggestion.tsx

@@ -1,4 +1,5 @@
 import React, { useMemo } from 'react';
+import Markdown from 'markdown-to-jsx';
 import { Message } from 'ui/Message/Message';
 
 const queryErrorsByLanguage: Record<
@@ -62,7 +63,7 @@ export const ErrorSuggestion: React.FunctionComponent<IProps> = ({
             iconSize={20}
             type="tip"
         >
-            <span dangerouslySetInnerHTML={{ __html: matchedError.message }} />
+            <Markdown>{matchedError.message}</Markdown>
         </Message>
     ) : null;
 };

datahub/webapp/components/DataDocStatementExecution/StatementMeta.tsx

@@ -1,22 +1,20 @@
 import React from 'react';
+import Markdown from 'markdown-to-jsx';
 import { Message } from 'ui/Message/Message';
 import { linkifyLog } from 'lib/utils';
 
 export const StatementMeta: React.FunctionComponent<{ metaInfo?: string }> = ({
     metaInfo,
 }) => {
-    metaInfo = metaInfo || '';
-    const linkifiedMetaInfo = React.useMemo(() => linkifyLog(metaInfo), [
+    const linkifiedMetaInfo = React.useMemo(() => linkifyLog(metaInfo ?? ''), [
         metaInfo,
     ]);
 
     return (
         metaInfo && (
             <div className="StatementMeta">
                 <Message className="StatementMeta-Message" size="small">
-                    <span
-                        dangerouslySetInnerHTML={{ __html: linkifiedMetaInfo }}
-                    />
+                    <Markdown>{linkifiedMetaInfo}</Markdown>
                 </Message>
             </div>
         )

datahub/webapp/components/Info/FAQ.tsx

@@ -4,8 +4,9 @@ import { Content } from 'ui/Content/Content';
 import { Title } from 'ui/Title/Title';
 
 import './FAQ.scss';
+import Markdown from 'markdown-to-jsx';
 
-const faqs = require('config/faqs.yaml').faqs;
+const faqs: Array<[string, string]> = require('config/faqs.yaml').faqs;
 
 export const FAQ: React.FunctionComponent = () => (
     <div className="FAQ m12">
@@ -17,7 +18,9 @@ export const FAQ: React.FunctionComponent = () => (
                         {question}
                     </Title>
                     <div className="FAQ-answer m12">
-                        <Content dangerouslySetInnerHTML={{ __html: answer }} />
+                        <Content>
+                            <Markdown>{answer}</Markdown>
+                        </Content>
                     </div>
                 </div>
             );

datahub/webapp/components/QueryEngineStatusViewer/QueryEngineStatusViewer.tsx

@@ -1,5 +1,6 @@
 import React, { useCallback, useEffect } from 'react';
 import { useSelector, useDispatch } from 'react-redux';
+import Markdown from 'markdown-to-jsx';
 
 import { titleize } from 'lib/utils';
 import { generateFormattedDate } from 'lib/utils/datetime';
@@ -94,7 +95,7 @@ export const QueryEngineStatusViewer: React.FC<IProps> = ({ engineId }) => {
 
         const messageDOMs = (data.messages ?? []).map((message, index) => (
             <Message key={index} type={messageType}>
-                <p dangerouslySetInnerHTML={{ __html: message }} />
+                <Markdown>{message}</Markdown>
             </Message>
         ));
         const messageSectionDOM = messageDOMs.length ? (

datahub/webapp/lib/markdown.ts

@@ -0,0 +1,42 @@
+/*
+We define a special format markdown such as
+
+---
+property_1: foo
+property_2: bar
+---
+**Hello** _World_
+
+This function would extract and remove the property part from markdown
+so we get:
+
+markdown: **Hello** _World_
+property: { property_1: "foo", property_2: "bar"}
+
+*/
+export function sanitizeAndExtraMarkdown(
+    markdown: string
+): [string, Record<string, string>] {
+    const lines = markdown.split('\n');
+    const filteredLines: string[] = [];
+    const properties: Record<string, string> = {};
+
+    // Remove --- blocks from markdown
+    let insidePropertyBlock = false;
+    for (const line of lines) {
+        if (line.startsWith('---')) {
+            insidePropertyBlock = !insidePropertyBlock;
+        } else {
+            if (insidePropertyBlock) {
+                const parts = line.split(':');
+                if (parts.length === 2) {
+                    properties[parts[0].trim()] = parts[1].trim();
+                }
+            } else {
+                filteredLines.push(line);
+            }
+        }
+    }
+
+    return [filteredLines.join('\n'), properties];
+}

datahub/webapp/ui/SmartForm/SmartForm.tsx

@@ -14,6 +14,7 @@ import { IconButton } from 'ui/Button/IconButton';
 import { ToggleSwitch } from 'ui/ToggleSwitch/ToggleSwitch';
 
 import './SmartForm.scss';
+import Markdown from 'markdown-to-jsx';
 
 export {
     updateValue,
@@ -93,7 +94,7 @@ function SimpleFormField<T>({
                 <FormFieldInputSection>{controlDOM}</FormFieldInputSection>
                 {helper ? (
                     <FormFieldHelpSection>
-                        <div dangerouslySetInnerHTML={{ __html: helper }} />
+                        <Markdown>{helper}</Markdown>
                     </FormFieldHelpSection>
                 ) : null}
             </FormField>

package.json

@@ -59,6 +59,7 @@
         "localforage": "^1.7.3",
         "lodash": "4.17.19",
         "lodash-decorators": "6.0.1",
+        "markdown-to-jsx": "^7.1.0",
         "memoize-one": "5.0.2",
         "mini-css-extract-plugin": "0.9.0",
         "moment": "2.24.0",
@@ -73,6 +74,7 @@
         "react-dnd": "^11.1.3",
         "react-dnd-html5-backend": "^11.1.3",
         "react-dom": "16.12.0",
+        "react-hot-loader": "4.12.19",
         "react-lazyload": "2.6.5",
         "react-redux": "7.1.3",
         "react-router-dom": "5.1.2",
@@ -84,7 +86,6 @@
         "redux": "4.0.4",
         "redux-logger": "3.0.6",
         "redux-thunk": "2.3.0",
-        "react-hot-loader": "4.12.19",
         "regenerator-runtime": "^0.13.2",
         "reselect": "4.0.0",
         "smooth-scroll-into-view-if-needed": "^1.1.27",
@@ -120,6 +121,7 @@
         "@types/jest": "^24.0.23",
         "@types/json-bigint": "^1.0.0",
         "@types/lodash": "4.14.116",
+        "@types/markdown-to-jsx": "^6.11.3",
         "@types/memoize-one": "3.1.1",
         "@types/qs": "^6.5.1",
         "@types/rc-time-picker": "^3.4.1",

yarn.lock

@@ -3279,6 +3279,13 @@
   resolved "https://registry.yarnpkg.com/@types/lodash/-/lodash-4.14.116.tgz#5ccf215653e3e8c786a58390751033a9adca0eb9"
   integrity sha512-lRnAtKnxMXcYYXqOiotTmJd74uawNWuPnsnPrrO7HiFuE3npE2iQhfABatbYDyxTNqZNuXzcKGhw37R7RjBFLg==
 
+"@types/markdown-to-jsx@^6.11.3":
+  version "6.11.3"
+  resolved "https://registry.yarnpkg.com/@types/markdown-to-jsx/-/markdown-to-jsx-6.11.3.tgz#cdd1619308fecbc8be7e6a26f3751260249b020e"
+  integrity sha512-30nFYpceM/ZEvhGiqWjm5quLUxNeld0HCzJEXMZZDpq53FPkS85mTwkWtCXzCqq8s5JYLgM5W392a02xn8Bdaw==
+  dependencies:
+    "@types/react" "*"
+
 "@types/[email protected]":
   version "3.1.1"
   resolved "https://registry.yarnpkg.com/@types/memoize-one/-/memoize-one-3.1.1.tgz#0e21a1b91dc031fb59c1e1657b807ca9aec77475"
@@ -11530,6 +11537,11 @@ markdown-to-jsx@^6.11.4:
     prop-types "^15.6.2"
     unquote "^1.1.0"
 
+markdown-to-jsx@^7.1.0:
+  version "7.1.0"
+  resolved "https://registry.yarnpkg.com/markdown-to-jsx/-/markdown-to-jsx-7.1.0.tgz#6f9b3be27560fab3f240ae253e63e52d811eb027"
+  integrity sha512-f0LxzbmLfInOSmTC/BaMB44RVmPN2cQom9D4pQzp6c5pOmO0yjxWMDJs78JSSLmncn+bi0Qy0VlJjsRVk8AUpg==
+
 md5.js@^1.3.4:
   version "1.3.5"
   resolved "https://registry.yarnpkg.com/md5.js/-/md5.js-1.3.5.tgz#b5d07b8e3216e3e27cd728d72f70d1e6a342005f"