ICU syntax parser

package.json

@@ -21,6 +21,7 @@
     "diff-match-patch": "^1.0.5",
     "draft-js": "^0.11.4",
     "flux": "^4.0.1",
+    "format-message-parse": "^6.2.4",
     "immutability-helper": "^3.1.1",
     "immutable": "^4.0.0-rc.12",
     "jquery": "^3.4.1",

public/css/sass/components/segment/IcuHighlight.scss

@@ -0,0 +1,26 @@
+@import '../../commons/colors';
+
+
+.icuItem {
+  display: inline-block;
+  position: relative;
+  color: $linkBlue;
+  &.icuItem-error {
+    color: $redDefault;
+    cursor: pointer;
+  }
+}
+
+.icu-tooltip {
+  padding: 6px;
+  max-width: 400px;
+  h3 {
+    font-size: 16px;
+    margin: 0;
+    margin-bottom: 4px;
+  }
+  p {
+    white-space: normal;
+  }
+}
+

public/css/sass/main.scss

@@ -24,6 +24,7 @@
 @import 'components/segment/Tag';
 @import 'components/segment/TooltipInfo';
 @import 'components/segment/Glossary';
+@import 'components/segment/IcuHighlight';
 @import 'components/segment/Editor';
 @import 'components/bulk-approve-bar/bulk_approve_bar';
 @import 'components/header/search';

public/js/cat_source/es6/components/segments/Editarea.js

@@ -24,7 +24,7 @@ import insertTag from './utils/DraftMatecatUtils/TagMenu/insertTag'
 import checkForMissingTags from './utils/DraftMatecatUtils/TagMenu/checkForMissingTag'
 import updateEntityData from './utils/DraftMatecatUtils/updateEntityData'
 import LexiqaUtils from '../../utils/lxq.main'
-import updateLexiqaWarnings from './utils/DraftMatecatUtils/updateLexiqaWarnings'
+import updateOffsetBasedOnEditorState from './utils/DraftMatecatUtils/updateOffsetBasedOnEditorState'
 import {tagSignatures} from './utils/DraftMatecatUtils/tagModel'
 import SegmentActions from '../../actions/SegmentActions'
 import getFragmentFromSelection from './utils/DraftMatecatUtils/DraftSource/src/component/handlers/edit/getFragmentFromSelection'
@@ -39,6 +39,11 @@ import {
   isSelectedEntity,
   getEntitiesSelected,
 } from './utils/DraftMatecatUtils/manageCaretPositionNearEntity'
+import {
+  createICUDecorator,
+  createIcuTokens,
+  isEqualICUTokens,
+} from './utils/DraftMatecatUtils/createICUDecorator'
 
 const {hasCommandModifier, isOptionKeyCommand, isCtrlKeyCommand} =
   KeyBindingUtil
@@ -65,7 +70,16 @@ class Editarea extends React.Component {
 
   constructor(props) {
     super(props)
-    const {onEntityClick, updateTagsInEditor, getUpdatedSegmentInfo} = this
+    const {onEntityClick, getUpdatedSegmentInfo} = this
+
+    const translation = this.props.translation
+
+    // If GuessTag is Enabled, clean translation from tags
+    const cleanTranslation = SegmentUtils.checkCurrentSegmentTPEnabled(
+      this.props.segment,
+    )
+      ? DraftMatecatUtils.removeTagsFromText(translation)
+      : translation
 
     this.decoratorsStructure = [
       {
@@ -83,16 +97,7 @@ class Editarea extends React.Component {
       },
     ]
     const decorator = new CompositeDecorator(this.decoratorsStructure)
-    //const decorator = new CompoundDecorator(this.decoratorsStructure);
-    // Escape html
-    const translation = this.props.translation
 
-    // If GuessTag is Enabled, clean translation from tags
-    const cleanTranslation = SegmentUtils.checkCurrentSegmentTPEnabled(
-      this.props.segment,
-    )
-      ? DraftMatecatUtils.removeTagsFromText(translation)
-      : translation
     // Inizializza Editor State con solo testo
     const plainEditorState = EditorState.createEmpty(decorator)
     const contentEncoded = DraftMatecatUtils.encodeContent(
@@ -121,6 +126,7 @@ class Editarea extends React.Component {
         [DraftMatecatConstants.LEXIQA_DECORATOR]: false,
         [DraftMatecatConstants.QA_BLACKLIST_DECORATOR]: false,
         [DraftMatecatConstants.SEARCH_DECORATOR]: false,
+        [DraftMatecatConstants.ICU_DECORATOR]: true,
       },
       previousSourceTagMap: null,
     }
@@ -136,9 +142,7 @@ class Editarea extends React.Component {
       this.updateTranslationInStore,
       100,
     )
-    this.updateTagsInEditorDebounced = debounce(updateTagsInEditor, 500)
     this.onCompositionStopDebounced = debounce(this.onCompositionStop, 1000)
-    this.focusEditorDebounced = debounce(this.focusEditor, 500)
   }
 
   getSearchParams = () => {
@@ -166,6 +170,15 @@ class Editarea extends React.Component {
     }
   }
 
+  addIcuDecorator = (tokens) => {
+    const newDecorator = createICUDecorator(tokens)
+    remove(
+      this.decoratorsStructure,
+      (decorator) => decorator.name === DraftMatecatConstants.ICU_DECORATOR,
+    )
+    this.decoratorsStructure.push(newDecorator)
+  }
+
   addSearchDecorator = () => {
     let {tagRange} = this.state
     let {searchParams, occurrencesInSearch, currentInSearchIndex} =
@@ -208,7 +221,10 @@ class Editarea extends React.Component {
       lxqDecodedTranslation,
       false,
     )
-    const updatedLexiqaWarnings = updateLexiqaWarnings(editorState, ranges)
+    const updatedLexiqaWarnings = updateOffsetBasedOnEditorState(
+      editorState,
+      ranges,
+    )
     if (updatedLexiqaWarnings.length > 0) {
       const newDecorator = DraftMatecatUtils.activateLexiqa(
         editorState,
@@ -411,6 +427,18 @@ class Editarea extends React.Component {
         changedDecorator = true
         this.removeDecorator(DraftMatecatConstants.SEARCH_DECORATOR)
       }
+      const contentState = editorState.getCurrentContent()
+      const plainText = contentState.getPlainText()
+      const icuTokens = createIcuTokens(plainText, editorState)
+      if (
+        !prevProps ||
+        !this.prevIcuTokens ||
+        !isEqualICUTokens(icuTokens, this.prevIcuTokens)
+      ) {
+        this.prevIcuTokens = icuTokens
+        changedDecorator = true
+        this.addIcuDecorator(icuTokens)
+      }
     } else {
       //Search
       if (
@@ -837,8 +865,8 @@ class Editarea extends React.Component {
             ? 'left'
             : 'right'
           : !isRTL
-          ? 'right'
-          : 'left'
+            ? 'right'
+            : 'left'
 
       const updatedStateNearZwsp = checkCaretIsNearZwsp({
         editorState: this.state.editorState,

public/js/cat_source/es6/components/segments/IcuHighlight.js

@@ -0,0 +1,27 @@
+import React, {useRef} from 'react'
+import Tooltip from '../common/Tooltip'
+
+export const IcuHighlight = ({start, end, tokens, children}) => {
+  const token = tokens.find((item) => item.start === start && item.end === end)
+  const refToken = useRef()
+  return (
+    <div
+      className={`icuItem ${token && token.type === 'error' ? 'icuItem-error' : ''}`}
+    >
+      {token.type === 'error' ? (
+        <Tooltip
+          content={
+            <div className="icu-tooltip">
+              <h3>ICU syntax error</h3>
+              <p>{token.message}</p>
+            </div>
+          }
+        >
+          <span ref={refToken}>{children}</span>
+        </Tooltip>
+      ) : (
+        <span>{children}</span>
+      )}
+    </div>
+  )
+}

public/js/cat_source/es6/components/segments/SegmentSource.js

@@ -12,14 +12,18 @@ import DraftMatecatUtils from './utils/DraftMatecatUtils'
 import * as DraftMatecatConstants from './utils/DraftMatecatUtils/editorConstants'
 import SegmentConstants from '../../constants/SegmentConstants'
 import LexiqaUtils from '../../utils/lxq.main'
-import updateLexiqaWarnings from './utils/DraftMatecatUtils/updateLexiqaWarnings'
+import updateOffsetBasedOnEditorState from './utils/DraftMatecatUtils/updateOffsetBasedOnEditorState'
 import getFragmentFromSelection from './utils/DraftMatecatUtils/DraftSource/src/component/handlers/edit/getFragmentFromSelection'
 import {getSplitPointTag} from './utils/DraftMatecatUtils/tagModel'
 import {SegmentContext} from './SegmentContext'
 import Assistant from '../icons/Assistant'
 import Education from '../icons/Education'
 import {TERM_FORM_FIELDS} from './SegmentFooterTabGlossary/SegmentFooterTabGlossary'
 import {getEntitiesSelected} from './utils/DraftMatecatUtils/manageCaretPositionNearEntity'
+import {
+  createICUDecorator,
+  createIcuTokens,
+} from './utils/DraftMatecatUtils/createICUDecorator'
 
 class SegmentSource extends React.Component {
   static contextType = SegmentContext
@@ -74,6 +78,7 @@ class SegmentSource extends React.Component {
         [DraftMatecatConstants.GLOSSARY_DECORATOR]: false,
         [DraftMatecatConstants.QA_GLOSSARY_DECORATOR]: false,
         [DraftMatecatConstants.SEARCH_DECORATOR]: false,
+        [DraftMatecatConstants.ICU_DECORATOR]: true,
       },
       isShowingOptionsToolbar: false,
     }
@@ -82,6 +87,8 @@ class SegmentSource extends React.Component {
       : 0
 
     this.delayAiAssistant
+
+    this.firstIcuCheck = false
   }
 
   getSearchParams = () => {
@@ -205,7 +212,10 @@ class SegmentSource extends React.Component {
       lxqDecodedSource,
       true,
     )
-    const updatedLexiqaWarnings = updateLexiqaWarnings(editorState, ranges)
+    const updatedLexiqaWarnings = updateOffsetBasedOnEditorState(
+      editorState,
+      ranges,
+    )
     if (updatedLexiqaWarnings.length > 0) {
       const newDecorator = DraftMatecatUtils.activateLexiqa(
         editorState,
@@ -224,6 +234,18 @@ class SegmentSource extends React.Component {
       this.removeDecorator(DraftMatecatConstants.LEXIQA_DECORATOR)
     }
   }
+  addIcuDecorator = () => {
+    const {editorState} = this.state
+    const contentState = editorState.getCurrentContent()
+    const plainText = contentState.getPlainText()
+    const tokens = createIcuTokens(plainText, editorState)
+    const newDecorator = createICUDecorator(tokens)
+    remove(
+      this.decoratorsStructure,
+      (decorator) => decorator.name === DraftMatecatConstants.ICU_DECORATOR,
+    )
+    this.decoratorsStructure.push(newDecorator)
+  }
 
   updateSourceInStore = () => {
     if (this.state.source !== '') {
@@ -333,6 +355,11 @@ class SegmentSource extends React.Component {
         changedDecorator = true
         this.removeDecorator(DraftMatecatConstants.SEARCH_DECORATOR)
       }
+      if (!this.firstIcuCheck) {
+        this.firstIcuCheck = true
+        changedDecorator = true
+        this.addIcuDecorator()
+      }
     } else {
       //Search
       if (
@@ -353,9 +380,8 @@ class SegmentSource extends React.Component {
         this.removeDecorator()
         ;(activeDecorators[DraftMatecatConstants.LEXIQA_DECORATOR] = false),
           (activeDecorators[DraftMatecatConstants.GLOSSARY_DECORATOR] = false),
-          (activeDecorators[
-            DraftMatecatConstants.QA_GLOSSARY_DECORATOR
-          ] = false),
+          (activeDecorators[DraftMatecatConstants.QA_GLOSSARY_DECORATOR] =
+            false),
           this.addSearchDecorator()
         activeDecorators[DraftMatecatConstants.SEARCH_DECORATOR] = true
         changedDecorator = true

public/js/cat_source/es6/components/segments/utils/DraftMatecatUtils/createICUDecorator.js

@@ -0,0 +1,65 @@
+import * as DraftMatecatConstants from './editorConstants'
+import parse from 'format-message-parse'
+import {IcuHighlight} from '../../IcuHighlight'
+import {isEqual} from 'lodash'
+import updateOffsetBasedOnEditorState from './updateOffsetBasedOnEditorState'
+export const createICUDecorator = (tokens = []) => {
+  return {
+    name: DraftMatecatConstants.ICU_DECORATOR,
+    strategy: (contentBlock, callback) => {
+      const currentText = contentBlock.getText()
+      tokens.forEach((token) => {
+        const subString = currentText.substring(token.start, token.end)
+        if (
+          token.end <= currentText.length &&
+          token.type !== 'text' &&
+          subString === token.text
+        ) {
+          callback(token.start, token.end)
+        }
+      })
+    },
+    component: IcuHighlight,
+    props: {
+      tokens,
+    },
+  }
+}
+
+export const createIcuTokens = (text, editorState) => {
+  const tokens = []
+  let error
+  try {
+    parse(text, {tokens: tokens})
+  } catch (e) {
+    error = {
+      type: 'error',
+      text: e.found,
+      start: e.column,
+      end: e.column + e.found.length,
+      message: e.message,
+    }
+    console.log(e)
+  }
+  let index = 0
+  const updatedTokens = tokens.map((token) => {
+    const value = {
+      type: token[0],
+      text: token[1],
+      start: index,
+      end: index + token[1].length,
+    }
+    index = index + token[1].length
+    return value
+  })
+  if (error) updatedTokens.push(error)
+  return updateOffsetBasedOnEditorState(editorState, updatedTokens)
+}
+
+export const isEqualICUTokens = (tokens, otherTokens) => {
+  const filterTokensFn = (token) => token.type !== 'text'
+  return isEqual(
+    tokens.filter(filterTokensFn),
+    otherTokens.filter(filterTokensFn),
+  )
+}

public/js/cat_source/es6/components/segments/utils/DraftMatecatUtils/editorConstants.js

@@ -8,3 +8,4 @@ export const QA_GLOSSARY_DECORATOR = 'qaCheckGlossary'
 export const QA_BLACKLIST_DECORATOR = 'qaCheckBlacklist'
 export const SEARCH_DECORATOR = 'search'
 export const SPLIT_DECORATOR = 'split'
+export const ICU_DECORATOR = 'icu'