feat: split text support multiple languages #593

Hufe921 · May 23, 2024 · 1cb07af · 1cb07af
1 parent bcf311f
commit 1cb07af
Showing 1 changed file with 21 additions and 13 deletions.
diff --git a/src/editor/utils/index.ts b/src/editor/utils/index.ts
@@ -107,19 +107,27 @@ export function getUUID(): string {
 
 export function splitText(text: string): string[] {
   const data: string[] = []
-  const symbolMap = new Map<number, string>()
-  for (const match of text.matchAll(UNICODE_SYMBOL_REG)) {
-    symbolMap.set(match.index!, match[0])
-  }
-  let t = 0
-  while (t < text.length) {
-    const symbol = symbolMap.get(t)
-    if (symbol) {
-      data.push(symbol)
-      t += symbol.length
-    } else {
-      data.push(text[t])
-      t++
+  if (Intl.Segmenter) {
+    const segmenter = new Intl.Segmenter()
+    const segments = segmenter.segment(text)
+    for (const { segment } of segments) {
+      data.push(segment)
+    }
+  } else {
+    const symbolMap = new Map<number, string>()
+    for (const match of text.matchAll(UNICODE_SYMBOL_REG)) {
+      symbolMap.set(match.index!, match[0])
+    }
+    let t = 0
+    while (t < text.length) {
+      const symbol = symbolMap.get(t)
+      if (symbol) {
+        data.push(symbol)
+        t += symbol.length
+      } else {
+        data.push(text[t])
+        t++
+      }
     }
   }
   return data