feat: use regex to enhance js engine support (#762)

shikijs · Sep 9, 2024 · ed36296 · ed36296
1 parent 4769588
commit ed36296
Show file tree

Hide file tree

Showing 8 changed files with 105 additions and 68 deletions.
diff --git a/.vscode/settings.json b/.vscode/settings.json
@@ -36,6 +36,7 @@
   ],
   "references.preferredLocation": "peek",
   "cSpell.words": [
+    "Extenerlaize",
     "shikijs"
   ]
 }
diff --git a/docs/references/engine-js-compat.md b/docs/references/engine-js-compat.md
diff --git a/package.json b/package.json
@@ -2,7 +2,7 @@
   "type": "module",
   "version": "1.16.2",
   "private": true,
-  "packageManager": "pnpm@9.9.0",
+  "packageManager": "pnpm@9.10.0",
   "scripts": {
     "lint": "eslint . --cache",
     "release": "bumpp && pnpm -r publish",
@@ -52,6 +52,7 @@
     "picocolors": "catalog:",
     "pnpm": "catalog:",
     "prettier": "catalog:",
+    "regex": "catalog:",
     "rimraf": "catalog:",
     "rollup": "catalog:",
     "rollup-plugin-copy": "catalog:",

diff --git a/packages/core/package.json b/packages/core/package.json
@@ -67,11 +67,12 @@
   },
   "dependencies": {
     "@shikijs/vscode-textmate": "catalog:",
-    "@types/hast": "catalog:"
+    "@types/hast": "catalog:",
+    "oniguruma-to-js": "catalog:",
+    "regex": "catalog:"
   },
   "devDependencies": {
     "hast-util-to-html": "catalog:",
-    "oniguruma-to-js": "catalog:",
     "vscode-oniguruma": "catalog:"
   }
 }
diff --git a/packages/core/rollup.config.mjs b/packages/core/rollup.config.mjs
@@ -35,6 +35,12 @@ const plugins = [
 const external = [
   'hast',
   '@shikijs/vscode-textmate',
+
+  // Externalize them to make it easier to patch and experiments
+  // Versions are pinned to avoid regressions
+  // Later we might consider to bundle them.
+  'oniguruma-to-js',
+  'regex',
 ]
 
 export default defineConfig([

diff --git a/packages/core/src/engines/javascript.ts b/packages/core/src/engines/javascript.ts
@@ -1,4 +1,5 @@
 import { onigurumaToRegexp } from 'oniguruma-to-js'
+import { rewrite } from 'regex'
 import type { JavaScriptRegexEngineOptions, PatternScanner, RegexEngine, RegexEngineString } from '../types'
 
 const MAX = 4294967295
@@ -7,10 +8,22 @@ const MAX = 4294967295
  * The default RegExp constructor for JavaScript regex engine.
  */
 export function defaultJavaScriptRegexConstructor(pattern: string): RegExp {
+  pattern = pattern
+    // YAML specific handling; TODO: move to tm-grammars
+    .replaceAll('[^\\s[-?:,\\[\\]{}#&*!|>\'"%@`]]', '[^\\s\\-?:,\\[\\]{}#&*!|>\'"%@`]')
+
+  const rewritten = rewrite(pattern, {
+    flags: 'dgm',
+    unicodeSetsPlugin: null,
+    disable: {
+      n: true,
+      v: true,
+      x: true,
+    },
+  })
+
   return onigurumaToRegexp(
-    pattern
-      // YAML specific handling; TODO: move to tm-grammars
-      .replaceAll('[^\\s[-?:,\\[\\]{}#&*!|>\'"%@`]]', '[^\\s\\-?:,\\[\\]{}#&*!|>\'"%@`]'),
+    rewritten.expression,
     {
       flags: 'dgm',
       ignoreContiguousAnchors: true,

diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml
@@ -51,11 +51,12 @@ catalog:
   minimist: ^1.2.8
   monaco-editor-core: ^0.51.0
   ofetch: ^1.3.4
-  oniguruma-to-js: ^0.3.3
+  oniguruma-to-js: 0.3.3
   picocolors: ^1.1.0
   pinia: ^2.2.2
-  pnpm: ^9.9.0
+  pnpm: ^9.10.0
   prettier: ^3.3.3
+  regex: 4.3.2
   rehype-raw: ^7.0.0
   rehype-stringify: ^10.0.0
   remark-parse: ^11.0.0