refactor(ui): Making improvements to UI ingestion forms, adding MySQL, Trino, Presto, MSSQL, MariaDB forms

datahub-web-react/src/app/ingest/source/builder/LookerWarning.tsx

@@ -14,13 +14,13 @@ export const LookerWarning = ({ type }: Props) => {
     if (type === LOOKER) {
         link = (
             <a href={LOOKML_DOC_LINK} target="_blank" rel="noopener noreferrer">
-                DataHub lookml module
+                DataHub LookML Ingestion Source
             </a>
         );
     } else if (type === LOOK_ML) {
         link = (
             <a href={LOOKER_DOC_LINK} target="_blank" rel="noopener noreferrer">
-                DataHub looker module
+                DataHub Looker Ingestion Source
             </a>
         );
     }
@@ -32,8 +32,8 @@ export const LookerWarning = ({ type }: Props) => {
             banner
             message={
                 <>
-                    To get complete Looker metadata integration (including Looker views and lineage to the underlying
-                    warehouse tables), you must <b>also</b> use the {link}.
+                    To complete the Looker integration (including Looker views and lineage to the underlying warehouse
+                    tables), you must <b>also</b> use the {link}.
                 </>
             }
         />

datahub-web-react/src/app/ingest/source/builder/RecipeForm/FormField.tsx

@@ -1,5 +1,5 @@
 import React from 'react';
-import { Checkbox, DatePicker, Form, Input, Select, Tooltip, FormInstance } from 'antd';
+import { Checkbox, DatePicker, Form, Input, Select, Tooltip } from 'antd';
 import styled from 'styled-components/macro';
 import Button from 'antd/lib/button';
 import { MinusCircleOutlined, PlusOutlined } from '@ant-design/icons';
@@ -94,11 +94,11 @@ interface Props {
     secrets: Secret[];
     refetchSecrets: () => void;
     removeMargin?: boolean;
-    form: FormInstance<any>;
+    updateFormValue: (field, value) => void;
 }
 
 function FormField(props: Props) {
-    const { field, secrets, refetchSecrets, removeMargin, form } = props;
+    const { field, secrets, refetchSecrets, removeMargin, updateFormValue } = props;
 
     if (field.type === FieldType.LIST) return <ListField field={field} removeMargin={removeMargin} />;
 
@@ -113,7 +113,7 @@ function FormField(props: Props) {
                 secrets={secrets}
                 removeMargin={removeMargin}
                 refetchSecrets={refetchSecrets}
-                form={form}
+                updateFormValue={updateFormValue}
             />
         );
 

datahub-web-react/src/app/ingest/source/builder/RecipeForm/RecipeForm.tsx

@@ -60,8 +60,9 @@ function getInitialValues(displayRecipe: string, allFields: any[]) {
     }
     if (recipeObj) {
         allFields.forEach((field) => {
-            initialValues[field.name] =
-                field.getValueFromRecipeOverride?.(recipeObj) || get(recipeObj, field.fieldPath);
+            initialValues[field.name] = field.getValueFromRecipeOverride
+                ? field.getValueFromRecipeOverride(recipeObj)
+                : get(recipeObj, field.fieldPath);
         });
     }
 
@@ -117,15 +118,28 @@ function RecipeForm(props: Props) {
         data?.listSecrets?.secrets.sort((secretA, secretB) => secretA.name.localeCompare(secretB.name)) || [];
     const [form] = Form.useForm();
 
+    function updateFormValue(fieldName, fieldValue) {
+        let updatedValues = YAML.parse(displayRecipe);
+        const recipeField = allFields.find((f) => f.name === fieldName);
+        if (recipeField) {
+            updatedValues = recipeField.setValueOnRecipeOverride
+                ? recipeField.setValueOnRecipeOverride(updatedValues, fieldValue)
+                : setFieldValueOnRecipe(updatedValues, fieldValue, recipeField.fieldPath);
+        }
+        const stagedRecipe = jsonToYaml(JSON.stringify(updatedValues));
+        setStagedRecipe(stagedRecipe);
+        form.setFieldsValue({ [fieldName]: fieldValue });
+    }
+
     function updateFormValues(changedValues: any, allValues: any) {
         let updatedValues = YAML.parse(displayRecipe);
 
         Object.keys(changedValues).forEach((fieldName) => {
             const recipeField = allFields.find((f) => f.name === fieldName);
             if (recipeField) {
-                updatedValues =
-                    recipeField.setValueOnRecipeOverride?.(updatedValues, allValues[fieldName]) ||
-                    setFieldValueOnRecipe(updatedValues, allValues[fieldName], recipeField.fieldPath);
+                updatedValues = recipeField.setValueOnRecipeOverride
+                    ? recipeField.setValueOnRecipeOverride(updatedValues, allValues[fieldName])
+                    : setFieldValueOnRecipe(updatedValues, allValues[fieldName], recipeField.fieldPath);
             }
         });
 
@@ -149,7 +163,7 @@ function RecipeForm(props: Props) {
                             secrets={secrets}
                             refetchSecrets={refetchSecrets}
                             removeMargin={i === fields.length - 1}
-                            form={form}
+                            updateFormValue={updateFormValue}
                         />
                     ))}
                     {CONNECTORS_WITH_TEST_CONNECTION.has(type as string) && (
@@ -187,7 +201,7 @@ function RecipeForm(props: Props) {
                                         secrets={secrets}
                                         refetchSecrets={refetchSecrets}
                                         removeMargin={i === filterFields.length - 1}
-                                        form={form}
+                                        updateFormValue={updateFormValue}
                                     />
                                 </MarginWrapper>
                             </>
@@ -213,7 +227,7 @@ function RecipeForm(props: Props) {
                             secrets={secrets}
                             refetchSecrets={refetchSecrets}
                             removeMargin={i === advancedFields.length - 1}
-                            form={form}
+                            updateFormValue={updateFormValue}
                         />
                     ))}
                 </Collapse.Panel>

datahub-web-react/src/app/ingest/source/builder/RecipeForm/SecretField/CreateSecretButton.tsx

@@ -24,11 +24,12 @@ const CreateButton = styled(Button)`
 `;
 
 interface Props {
+    initialState?: SecretBuilderState;
     onSubmit?: (state: SecretBuilderState) => void;
     refetchSecrets: () => void;
 }
 
-function CreateSecretButton({ onSubmit, refetchSecrets }: Props) {
+function CreateSecretButton({ initialState, onSubmit, refetchSecrets }: Props) {
     const [isCreateModalVisible, setIsCreateModalVisible] = useState(false);
     const [createSecretMutation] = useCreateSecretMutation();
 
@@ -62,6 +63,7 @@ function CreateSecretButton({ onSubmit, refetchSecrets }: Props) {
             </CreateButton>
             {isCreateModalVisible && (
                 <SecretBuilderModal
+                    initialState={initialState}
                     visible={isCreateModalVisible}
                     onCancel={() => setIsCreateModalVisible(false)}
                     onSubmit={createSecret}

datahub-web-react/src/app/ingest/source/builder/RecipeForm/SecretField/SecretField.tsx

@@ -1,5 +1,5 @@
 import React, { ReactNode } from 'react';
-import { AutoComplete, Divider, Form, FormInstance } from 'antd';
+import { AutoComplete, Divider, Form } from 'antd';
 import styled from 'styled-components/macro';
 import { Secret } from '../../../../../../types.generated';
 import CreateSecretButton from './CreateSecretButton';
@@ -52,7 +52,7 @@ interface SecretFieldProps {
     secrets: Secret[];
     removeMargin?: boolean;
     refetchSecrets: () => void;
-    form: FormInstance<any>;
+    updateFormValue: (field, value) => void;
 }
 
 function SecretFieldTooltip({ tooltipLabel }: { tooltipLabel?: string | ReactNode }) {
@@ -84,7 +84,7 @@ const encodeSecret = (secretName: string) => {
     return `\${${secretName}}`;
 };
 
-function SecretField({ field, secrets, removeMargin, form, refetchSecrets }: SecretFieldProps) {
+function SecretField({ field, secrets, removeMargin, updateFormValue, refetchSecrets }: SecretFieldProps) {
     const options = secrets.map((secret) => ({ value: encodeSecret(secret.name), label: secret.name }));
 
     return (
@@ -108,9 +108,7 @@ function SecretField({ field, secrets, removeMargin, form, refetchSecrets }: Sec
                             {menu}
                             <StyledDivider />
                             <CreateSecretButton
-                                onSubmit={(state) =>
-                                    form.setFields([{ name: field.name, value: encodeSecret(state.name as string) }])
-                                }
+                                onSubmit={(state) => updateFormValue(field.name, encodeSecret(state.name as string))}
                                 refetchSecrets={refetchSecrets}
                             />
                         </>

datahub-web-react/src/app/ingest/source/builder/RecipeForm/bigquery.ts

@@ -1,19 +1,20 @@
-import { RecipeField, FieldType, setListValuesOnRecipe } from './common';
+import { RecipeField, FieldType } from './common';
 
 export const BIGQUERY_PROJECT_ID: RecipeField = {
     name: 'project_id',
-    label: 'BigQuery Project ID',
+    label: 'Project ID',
     tooltip: 'Project ID where you have rights to run queries and create tables.',
     placeholder: 'my-project-123',
     type: FieldType.TEXT,
     fieldPath: 'source.config.project_id',
     rules: null,
+    required: true,
 };
 
 export const BIGQUERY_CREDENTIAL_PROJECT_ID: RecipeField = {
     name: 'credential.project_id',
     label: 'Credentials Project ID',
-    tooltip: 'Project id to set the credentials.',
+    tooltip: "The Project ID, which can be found in your service account's JSON Key (project_id)",
     placeholder: 'my-project-123',
     type: FieldType.TEXT,
     fieldPath: 'source.config.credential.project_id',
@@ -23,129 +24,43 @@ export const BIGQUERY_CREDENTIAL_PROJECT_ID: RecipeField = {
 export const BIGQUERY_PRIVATE_KEY_ID: RecipeField = {
     name: 'credential.private_key_id',
     label: 'Private Key Id',
-    tooltip: 'Private key id.',
-    placeholder: 'BQ_PRIVATE_KEY_ID',
+    tooltip: "The Private Key id, which can be found in your service account's JSON Key (private_key_id)",
     type: FieldType.SECRET,
     fieldPath: 'source.config.credential.private_key_id',
+    placeholder: 'd0121d0000882411234e11166c6aaa23ed5d74e0',
     rules: null,
+    required: true,
 };
 
 export const BIGQUERY_PRIVATE_KEY: RecipeField = {
     name: 'credential.private_key',
     label: 'Private Key',
-    placeholder: 'BQ_PRIVATE_KEY',
-    tooltip: 'Private key in a form of "-----BEGIN PRIVATE KEY-----\nprivate-key\n-----END PRIVATE KEY-----\n".',
+    tooltip: "The Private key, which can be found in your service account's JSON Key (private_key).",
+    placeholder: '-----BEGIN PRIVATE KEY-----....\n-----END PRIVATE KEY-----',
     type: FieldType.SECRET,
     fieldPath: 'source.config.credential.private_key',
     rules: null,
+    required: true,
 };
 
 export const BIGQUERY_CLIENT_EMAIL: RecipeField = {
     name: 'credential.client_email',
     label: 'Client Email',
-    tooltip: 'Client email.',
+    tooltip: "The Client Email, which can be found in your service account's JSON Key (client_email).",
     placeholder: '[email protected]',
     type: FieldType.TEXT,
     fieldPath: 'source.config.credential.client_email',
     rules: null,
+    required: true,
 };
 
 export const BIGQUERY_CLIENT_ID: RecipeField = {
     name: 'credential.client_id',
     label: 'Client ID',
-    tooltip: 'Client ID.',
+    tooltip: "The Client ID, which can be found in your service account's JSON Key (client_id).",
     placeholder: '123456789098765432101',
     type: FieldType.TEXT,
     fieldPath: 'source.config.credential.client_id',
     rules: null,
-};
-
-const schemaAllowFieldPath = 'source.config.schema_pattern.allow';
-export const BIGQUERY_SCHEMA_ALLOW: RecipeField = {
-    name: 'schema_pattern.allow',
-    label: 'Allow Patterns',
-    tooltip: 'Use regex here.',
-    placeholder: '^my_schema$',
-    type: FieldType.LIST,
-    buttonLabel: 'Add pattern',
-    fieldPath: schemaAllowFieldPath,
-    rules: null,
-    section: 'Schemas',
-    setValueOnRecipeOverride: (recipe: any, values: string[]) =>
-        setListValuesOnRecipe(recipe, values, schemaAllowFieldPath),
-};
-
-const schemaDenyFieldPath = 'source.config.schema_pattern.deny';
-export const BIGQUERY_SCHEMA_DENY: RecipeField = {
-    name: 'schema_pattern.deny',
-    label: 'Deny Patterns',
-    tooltip: 'Use regex here.',
-    placeholder: '^my_schema$',
-    type: FieldType.LIST,
-    buttonLabel: 'Add pattern',
-    fieldPath: schemaDenyFieldPath,
-    rules: null,
-    section: 'Schemas',
-    setValueOnRecipeOverride: (recipe: any, values: string[]) =>
-        setListValuesOnRecipe(recipe, values, schemaDenyFieldPath),
-};
-
-const tableAllowFieldPath = 'source.config.table_pattern.allow';
-export const BIGQUERY_TABLE_ALLOW: RecipeField = {
-    name: 'table_pattern.allow',
-    label: 'Allow Patterns',
-    tooltip: 'Use regex here.',
-    placeholder: '^my_schema\\.table_name$',
-    type: FieldType.LIST,
-    buttonLabel: 'Add pattern',
-    fieldPath: tableAllowFieldPath,
-    rules: null,
-    section: 'Tables',
-    setValueOnRecipeOverride: (recipe: any, values: string[]) =>
-        setListValuesOnRecipe(recipe, values, tableAllowFieldPath),
-};
-
-const tableDenyFieldPath = 'source.config.table_pattern.deny';
-export const BIGQUERY_TABLE_DENY: RecipeField = {
-    name: 'table_pattern.deny',
-    label: 'Deny Patterns',
-    tooltip: 'Use regex here.',
-    placeholder: '^my_schema\\.table_name$',
-    type: FieldType.LIST,
-    buttonLabel: 'Add pattern',
-    fieldPath: tableDenyFieldPath,
-    rules: null,
-    section: 'Tables',
-    setValueOnRecipeOverride: (recipe: any, values: string[]) =>
-        setListValuesOnRecipe(recipe, values, tableDenyFieldPath),
-};
-
-const viewAllowFieldPath = 'source.config.view_pattern.allow';
-export const BIGQUERY_VIEW_ALLOW: RecipeField = {
-    name: 'view_pattern.allow',
-    label: 'Allow Patterns',
-    tooltip: 'Use regex here.',
-    placeholder: '^my_schema\\.view_name$',
-    type: FieldType.LIST,
-    buttonLabel: 'Add pattern',
-    fieldPath: viewAllowFieldPath,
-    rules: null,
-    section: 'Views',
-    setValueOnRecipeOverride: (recipe: any, values: string[]) =>
-        setListValuesOnRecipe(recipe, values, viewAllowFieldPath),
-};
-
-const viewDenyFieldPath = 'source.config.view_pattern.deny';
-export const BIGQUERY_VIEW_DENY: RecipeField = {
-    name: 'view_pattern.deny',
-    label: 'Deny Patterns',
-    tooltip: 'Use regex here.',
-    placeholder: '^my_schema\\.view_name$',
-    type: FieldType.LIST,
-    buttonLabel: 'Add pattern',
-    fieldPath: viewDenyFieldPath,
-    rules: null,
-    section: 'Views',
-    setValueOnRecipeOverride: (recipe: any, values: string[]) =>
-        setListValuesOnRecipe(recipe, values, viewDenyFieldPath),
+    required: true,
 };

datahub-web-react/src/app/ingest/source/builder/RecipeForm/bigqueryBeta.ts

@@ -3,11 +3,12 @@ import { FieldType, RecipeField, setListValuesOnRecipe } from './common';
 export const BIGQUERY_BETA_PROJECT_ID: RecipeField = {
     name: 'credential.project_id',
     label: 'Project ID',
-    tooltip: 'Project id to set the credentials.',
+    tooltip: "The Project ID, which can be found in your service account's JSON Key (project_id)",
     placeholder: 'my-project-123',
     type: FieldType.TEXT,
     fieldPath: 'source.config.credential.project_id',
     rules: null,
+    required: true,
 };
 
 const projectIdAllowFieldPath = 'source.config.project_id_pattern.allow';