Merge pull request #377 from eccenca/bugfix/emojis-support-CMEM-7217

Fix emoji false-positives in invisible character detection

Author: andreas-schultz

.typescript/tsbuild-esm.json

@@ -1,7 +1,7 @@
 {
     "extends": "./../tsconfig.json",
     "compilerOptions": {
-        "lib": ["dom", "dom.iterable", "es2015", "es2020", "es2021", "es2015.collection", "es2015.iterable"],
+        "lib": ["dom", "dom.iterable", "es2015", "es2020", "es2021", "es2022.intl", "es2015.collection", "es2015.iterable"],
         "module": "es2015",
         "target": "es5",
         "noEmit": false,

CHANGELOG.md

@@ -48,6 +48,8 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/) and this p
     -   adjust displaying fallback symbols in different browsers
 -   `<CodeMirror />`
     -   use the latest provided `onChange` function
+-   `<TextField />`, `<TextArea />`
+    -  fix emoji false-positives in invisible character detection
 
 ### Changed
 

src/components/TextField/stories/TextField.stories.tsx

@@ -61,3 +61,26 @@ const invisibleCharacterWarningProps: TextFieldProps = {
     defaultValue: "Invisible character ->​<-",
 };
 InvisibleCharacterWarning.args = invisibleCharacterWarningProps;
+
+/** Text field showing that emoji (✔️ variation-selector, 👨‍👩‍👧‍👦 ZWJ,  #️⃣ keycap)
+ * are NOT reported as invisible characters, while a genuine ZWS still is. */
+export const InvisibleCharacterWarningWithEmoji = Template.bind({});
+
+const invisibleCharacterWarningWithEmojiProps: TextFieldProps = {
+    ...Default.args,
+    invisibleCharacterWarning: {
+        callback: (codePoints) => {
+            if (codePoints.size) {
+                const codePointsString = [...codePoints]
+                    .map((n) => characters.invisibleZeroWidthCharacters.codePointMap.get(n)?.fullLabel)
+                    .join(", ");
+                alert("Invisible character detected in input string. Code points: " + codePointsString);
+            }
+        },
+        callbackDelay: 500,
+    },
+    onChange: () => {},
+    // ZWS should be flagged; ✔️ 👨‍👩‍👧‍👦 #️⃣ should NOT be flagged
+    defaultValue: "Check\u200B ✔️ 👨‍👩‍👧‍👦 #️⃣",
+};
+InvisibleCharacterWarningWithEmoji.args = invisibleCharacterWarningWithEmojiProps;

src/components/TextField/tests/useTextValidation.test.tsx

@@ -0,0 +1,83 @@
+import React from "react";
+import { act, render } from "@testing-library/react";
+
+import { useTextValidation } from "../useTextValidation";
+
+const HookWrapper: React.FC<{ value: string; callback: jest.Mock; callbackDelay?: number }> = ({
+    value,
+    callback,
+    callbackDelay = 0,
+}) => {
+    useTextValidation({
+        value,
+        onChange: jest.fn(),
+        invisibleCharacterWarning: { callback, callbackDelay },
+    });
+    return null;
+};
+
+describe("useTextValidation", () => {
+    beforeEach(() => {
+        jest.useFakeTimers();
+    });
+
+    afterEach(() => {
+        jest.useRealTimers();
+    });
+
+    /** Render the hook with a controlled value and flush the debounce timer. */
+    const runWithValue = (value: string, callbackDelay = 0) => {
+        const callback = jest.fn();
+        render(<HookWrapper value={value} callback={callback} callbackDelay={callbackDelay} />);
+        act(() => {
+            jest.runAllTimers();
+        });
+        return callback;
+    };
+
+    describe("invisible character detection", () => {
+        it("reports empty set for plain text", () => {
+            const callback = runWithValue("hello world");
+            expect(callback).toHaveBeenCalledWith(new Set());
+        });
+
+        it("detects zero-width space (U+200B)", () => {
+            const callback = runWithValue("hello\u200Bworld");
+            expect(callback).toHaveBeenCalledWith(new Set([0x200b]));
+        });
+
+        it("detects zero-width non-joiner (U+200C)", () => {
+            const callback = runWithValue("hello\u200Cworld");
+            expect(callback).toHaveBeenCalledWith(new Set([0x200c]));
+        });
+    });
+
+    describe("emoji false-positive prevention", () => {
+        it("does not flag ✔️ (base char + variation selector U+FE0F)", () => {
+            const callback = runWithValue("✔️");
+            expect(callback).toHaveBeenCalledWith(new Set());
+        });
+
+        it("does not flag ZWJ sequence emoji 👨‍👩‍👧‍👦", () => {
+            const callback = runWithValue("👨‍👩‍👧‍👦");
+            expect(callback).toHaveBeenCalledWith(new Set());
+        });
+
+        it("does not flag keycap emoji #️⃣", () => {
+            const callback = runWithValue("#️⃣");
+            expect(callback).toHaveBeenCalledWith(new Set());
+        });
+    });
+
+    describe("mixed content", () => {
+        it("detects ZWS while ignoring surrounding emoji", () => {
+            const callback = runWithValue("Check\u200B ✔️👨‍👩‍👧‍#️⃣");
+            expect(callback).toHaveBeenCalledWith(new Set([0x200b]));
+        });
+
+        it("reports empty set for text with only emoji", () => {
+            const callback = runWithValue("✔️ 👨‍👩‍👧‍👦#️⃣");
+            expect(callback).toHaveBeenCalledWith(new Set());
+        });
+    });
+});

src/components/TextField/useTextValidation.ts

@@ -44,19 +44,28 @@ export const useTextValidation = <T>({ value, onChange, invisibleCharacterWarnin
         state.current.detectedCodePoints = new Set();
     }, []);
     const detectionRegex = React.useMemo(() => chars.invisibleZeroWidthCharacters.createRegex(), []);
+    const segmenter = React.useMemo(() => new Intl.Segmenter(undefined, { granularity: "grapheme" }), []);
+    const emojiRegex = React.useMemo(() => new RegExp("\\p{Extended_Pictographic}|\\u20E3", "u"), []);
+
     const detectIssues = React.useCallback(
         (value: string): void => {
-            detectionRegex.lastIndex = 0;
-            let matchArray = detectionRegex.exec(value);
-            while (matchArray) {
-                const codePoint = matchArray[0].codePointAt(0);
-                if (codePoint) {
-                    state.current.detectedCodePoints.add(codePoint);
+            for (const { segment } of segmenter.segment(value)) {
+                if (emojiRegex.test(segment)) {
+                    // skip emoji clusters since they legitimately contain variation selectors, ZWJ, tags, etc.
+                } else {
+                    detectionRegex.lastIndex = 0;
+                    let matchArray = detectionRegex.exec(segment);
+                    while (matchArray) {
+                        const codePoint = matchArray[0].codePointAt(0);
+                        if (codePoint) {
+                            state.current.detectedCodePoints.add(codePoint);
+                        }
+                        matchArray = detectionRegex.exec(segment);
+                    }
                 }
-                matchArray = detectionRegex.exec(value);
             }
         },
-        [detectionRegex]
+        [detectionRegex, segmenter, emojiRegex]
     );
     // Checks if the value contains any problematic characters with a small delay.
     const checkValue = React.useCallback(