[FIX] cell: keep non-breaking spaces

The non-breaking spaces were removed when updating a cell. This caused
issues is odoo pivots, since the formula contained a group name with
the non-breaking space removed, but the data was still using the
non-breaking space.

closes #5365

Task: 4403607
X-original-commit: 5719d7d
Signed-off-by: Adrien Minne (adrm) <[email protected]>
Signed-off-by: Rémi Rahir (rar) <[email protected]>

src/formulas/tokenizer.ts

@@ -3,7 +3,8 @@ import {
   TokenizingChars,
   getFormulaNumberRegex,
   rangeReference,
-  replaceSpecialSpaces,
+  replaceNewLines,
+  whiteSpaceRegexp,
 } from "../helpers/index";
 import { DEFAULT_LOCALE, Locale } from "../types";
 import { CellErrorType } from "../types/errors";
@@ -48,7 +49,7 @@ export interface Token {
 }
 
 export function tokenize(str: string, locale = DEFAULT_LOCALE): Token[] {
-  str = replaceSpecialSpaces(str);
+  str = replaceNewLines(str);
   const chars = new TokenizingChars(str);
   const result: Token[] = [];
 
@@ -214,13 +215,13 @@ function tokenizeSpace(chars: TokenizingChars): Token | null {
     return { type: "SPACE", value: NEWLINE.repeat(length) };
   }
 
-  while (chars.current === " ") {
-    length++;
-    chars.shift();
+  let spaces = "";
+  while (chars.current && chars.current.match(whiteSpaceRegexp)) {
+    spaces += chars.shift();
   }
 
-  if (length) {
-    return { type: "SPACE", value: " ".repeat(length) };
+  if (spaces) {
+    return { type: "SPACE", value: spaces };
   }
   return null;
 }

src/helpers/misc.ts

@@ -444,6 +444,7 @@ export function removeFalsyAttributes<T extends Object | undefined | null>(obj:
  * https://developer.mozilla.org/en-US/docs/Web/JavaScript/Guide/Regular_Expressions/Character_Classes
  */
 const whiteSpaceSpecialCharacters = [
+  " ",
   "\t",
   "\f",
   "\v",
@@ -458,16 +459,15 @@ const whiteSpaceSpecialCharacters = [
   String.fromCharCode(parseInt("3000", 16)),
   String.fromCharCode(parseInt("feff", 16)),
 ];
-const whiteSpaceRegexp = new RegExp(whiteSpaceSpecialCharacters.join("|") + "|(\r\n|\r|\n)", "g");
+export const whiteSpaceRegexp = new RegExp(whiteSpaceSpecialCharacters.join("|"), "g");
+const newLineRegexp = /(\r\n|\r)/g;
 
 /**
- * Replace all the special spaces in a string (non-breaking, tabs, ...) by normal spaces, and all the
- * different newlines types by \n.
+ * Replace all different newlines characters by \n
  */
-export function replaceSpecialSpaces(text: string | undefined): string {
+export function replaceNewLines(text: string | undefined): string {
   if (!text) return "";
-  if (!whiteSpaceRegexp.test(text)) return text;
-  return text.replace(whiteSpaceRegexp, (match, newLine) => (newLine ? NEWLINE : " "));
+  return text.replace(newLineRegexp, NEWLINE);
 }
 
 /**

src/plugins/core/cell.ts

@@ -15,7 +15,7 @@ import {
   detectNumberFormat,
   isInside,
   range,
-  replaceSpecialSpaces,
+  replaceNewLines,
   toCartesian,
   toXC,
 } from "../../helpers/index";
@@ -564,7 +564,7 @@ export class CellPlugin extends CorePlugin<CoreState> implements CoreState {
     const hasContent = "content" in after || "formula" in after;
 
     // Compute the new cell properties
-    const afterContent = hasContent ? replaceSpecialSpaces(after?.content) : before?.content || "";
+    const afterContent = hasContent ? replaceNewLines(after?.content) : before?.content || "";
     let style: Style | undefined;
     if (after.style !== undefined) {
       style = after.style || undefined;

tests/cells/cell_plugin.test.ts

@@ -243,6 +243,12 @@ describe("getCellText", () => {
     setCellContent(model, "A1", '="hello \\"world\\""');
     expect(getEvaluatedCell(model, "A1")?.formattedValue).toBe('hello "world"');
   });
+
+  test("Non breaking spaces are kept on cell insertion", () => {
+    const model = new Model();
+    setCellContent(model, "A1", "hello\u00A0world");
+    expect(getCellText(model, "A1")).toBe("hello\u00A0world");
+  });
 });
 
 describe("link cell", () => {
@@ -549,7 +555,6 @@ test.each([
   ["Line\nLine2", "Line\nLine2"],
   ["Line\rLine2", "Line\nLine2"],
   ["Line\r\nLine2", "Line\nLine2"],
-  ["Word\xa0Word2", "Word Word2"], // \xa0 => non-breaking space
 ])(
   "Content string given to update cell are properly sanitized %s",
   (originalString: string, sanitizedString: string) => {

tests/evaluation/tokenizer.test.ts

@@ -60,7 +60,7 @@ describe("tokenizer", () => {
 
   test("spaces", () => {
     expect(tokenize("  ")).toEqual([{ type: "SPACE", value: " ".repeat(2) }]);
-    expect(tokenize(" \t\v\f")).toEqual([{ type: "SPACE", value: " ".repeat(4) }]);
+    expect(tokenize(" \t\v\f")).toEqual([{ type: "SPACE", value: " \t\v\f" }]);
     expect(tokenize("= 50 %")).toEqual([
       { type: "OPERATOR", value: "=" },
       { type: "SPACE", value: " " },