add support for international character languages

rodrigopivi · Aug 23, 2018 · 99dd4e6 · 99dd4e6
1 parent 1bb2277
commit 99dd4e6
Show file tree

Hide file tree

Showing 4 changed files with 132 additions and 10 deletions.
diff --git a/parser/chatito.pegjs b/parser/chatito.pegjs
@@ -5,8 +5,8 @@ TopLevelStatement = od:(IntentDefinition/SlotDefinition/AliasDefinition) { retur
 
 // ============= Entities =============
 EntityOpt = "?"
-EntityBody = "[" value:KeywordLiteral "]" { return value }
-EntityOptionalBody = "[" value:KeywordLiteral opt:EntityOpt? "]"
+EntityBody = "[" value:EntityKeywordLiteral "]" { return value }
+EntityOptionalBody = "[" value:EntityKeywordLiteral opt:EntityOpt? "]"
     { return { value: value, opt: !!opt  }; }
 BasicKeywordLiterals = value:AnyTextWithAlias { return { value: value, type: "Text" }}
 // Entities (slot and aliases) allow any text except end of lines and alias definitions
@@ -30,10 +30,10 @@ IntentDefinition = EOL? o:EntityIntentDefinition EOL
 
 // Slot
 SlotVariationStartDefinition = "#"
-SlotVariationDefinition = SlotVariationStartDefinition id:KeywordLiteral { return id }
-EntitySlotDefinition = "@[" value:KeywordLiteral variation:SlotVariationDefinition? "]" args:EntityArguments?
+SlotVariationDefinition = SlotVariationStartDefinition id:SlotKeywordLiteral { return id }
+EntitySlotDefinition = "@[" value:SlotKeywordLiteral variation:SlotVariationDefinition? "]" args:EntityArguments?
     { return { value: value, type: "SlotDefinition", variation: variation, args: args, location: location() } }
-SlotOptionalBody = "[" value:KeywordLiteral variation:SlotVariationDefinition? opt:EntityOpt? "]"
+SlotOptionalBody = "[" value:SlotKeywordLiteral variation:SlotVariationDefinition? opt:EntityOpt? "]"
     { return { value: value, opt: !!opt, variation: variation }; }
 OptionalSlot = "@" op:SlotOptionalBody
     { return { value: op.value, type: "Slot", opt: op.opt, location: location(), variation: op.variation } }
@@ -55,7 +55,11 @@ Dedent = &{ level--; return true; }
 // ============= Primitives =============
 AnyTextWithoutEOL = v:(t:((!"\r\n")(!"\n") .) { return t.join(""); })+ { return v.join(""); }
 CommentLine = EOL? "//" c:AnyTextWithoutEOL EOS? { return { type: "Comment" , value: c.trim() }; }
-KeywordLiteral "word" = v:([a-zA-Z0-9_ \:\+]+) { return v.join(""); }
+
+// KeywordLiteral "word" = v:([a-zA-Z0-9_ \:\+]+) { return v.join(""); }
+EntityKeywordLiteral "entity name" = v:(t:((!"\r\n")(!"\n")(!"]")(!"?") .) { return t.join(""); })+ { return v.join(""); }
+SlotKeywordLiteral "entity name" = v:(t:((!"\r\n")(!"\n")(!"#")(!"]")(!"?") .) { return t.join(""); })+ { return v.join(""); }
+
 Integer "integer" = [0-9]+ { return parseInt(text(), 10); }
 EOS "end of sentence" = EOL / EOF
 EOL "end of line "= (EOLNonWindows/EOLWindows)+

diff --git a/spec.md b/spec.md
@@ -68,7 +68,7 @@ There are three types of entities: `intent`, `slot` and `alias`.
 
 The intent entity is defined by the `%[` symbols at the start of a line, following by the entity name and `]`.
 
-Entity names should be at least 1 character long and can contain any characters except `]`, `line end` and `?`
+Intent names should be at least 1 character long and can contain any characters except `]`, `line end` and `?`
 . e.g.: (%[intentName], %[intent_name], %[intent name])
 
 Repeating intent name definitions should not be allowed.

diff --git a/src/tests/__snapshots__/parser.spec.ts.snap b/src/tests/__snapshots__/parser.spec.ts.snap
@@ -453,7 +453,98 @@ exports[`Example with comments spec CORRECT parser output for exampleWithCorrect
 ]"
 `;
 
-exports[`Example with comments spec CORRECT parser output for exampleWithCorrectComments 2`] = `null`;
+exports[`Example with comments spec CORRECT parser output for exampleWithWrongComments 1`] = `
+Object {
+  "error": [SyntaxError: Expected "%", "//", "@[", or "~" but " " found.],
+  "location": Object {
+    "column": 1,
+    "line": 2,
+  },
+}
+`;
+
+exports[`Example with international language characters CORRECT parser output 1`] = `
+"[
+  {
+    \\"type\\": \\"IntentDefinition\\",
+    \\"key\\": \\"中文\\",
+    \\"args\\": null,
+    \\"location\\": {
+      \\"start\\": {
+        \\"offset\\": 1,
+        \\"line\\": 2,
+        \\"column\\": 1
+      },
+      \\"end\\": {
+        \\"offset\\": 6,
+        \\"line\\": 2,
+        \\"column\\": 6
+      }
+    },
+    \\"inner\\": [
+      [
+        {
+          \\"value\\": \\"中文 \\",
+          \\"type\\": \\"Text\\"
+        },
+        {
+          \\"value\\": \\"中文\\",
+          \\"type\\": \\"Slot\\",
+          \\"opt\\": false,
+          \\"location\\": {
+            \\"start\\": {
+              \\"offset\\": 14,
+              \\"line\\": 3,
+              \\"column\\": 8
+            },
+            \\"end\\": {
+              \\"offset\\": 19,
+              \\"line\\": 3,
+              \\"column\\": 13
+            }
+          },
+          \\"variation\\": null
+        },
+        {
+          \\"value\\": \\" \\",
+          \\"type\\": \\"Text\\"
+        },
+        {
+          \\"value\\": \\"中文\\",
+          \\"type\\": \\"Alias\\",
+          \\"opt\\": false
+        }
+      ]
+    ]
+  },
+  {
+    \\"type\\": \\"SlotDefinition\\",
+    \\"key\\": \\"中文\\",
+    \\"args\\": null,
+    \\"location\\": {
+      \\"start\\": {
+        \\"offset\\": 27,
+        \\"line\\": 5,
+        \\"column\\": 1
+      },
+      \\"end\\": {
+        \\"offset\\": 32,
+        \\"line\\": 5,
+        \\"column\\": 6
+      }
+    },
+    \\"inner\\": [
+      [
+        {
+          \\"value\\": \\"中文\\",
+          \\"type\\": \\"Text\\"
+        }
+      ]
+    ],
+    \\"variation\\": null
+  }
+]"
+`;
 
 exports[`Example with multi intent CORRECT parser output 1`] = `
 "[

diff --git a/src/tests/parser.spec.ts b/src/tests/parser.spec.ts
@@ -306,11 +306,11 @@ describe('Example with comments spec', () => {
     3 days
     5 hours
 `;
-    test('CORRECT parser output for exampleWithCorrectComments', () => {
+    test('CORRECT parser output for exampleWithWrongComments', () => {
         let error: any = null;
         let result = null;
         try {
-            result = chatitoParser.parse(exampleWithCorrectComments);
+            result = chatitoParser.parse(exampleWithWrongComments);
         } catch (e) {
             error = { error: e };
             if (e.location) {
@@ -323,3 +323,30 @@ describe('Example with comments spec', () => {
         expect(error).toMatchSnapshot();
     });
 });
+
+describe('Example with international language characters', () => {
+    const slotExamplesWithWeirdKeywords = `
+%[中文]
+    中文 @[中文] ~[中文]
+
+@[中文]
+    中文
+`;
+    test('CORRECT parser output', () => {
+        let error: any = null;
+        let result = null;
+        try {
+            result = chatitoParser.parse(slotExamplesWithWeirdKeywords);
+        } catch (e) {
+            error = { error: e };
+            if (e.location) {
+                error.location = {
+                    line: e.location.start.line,
+                    column: e.location.start.column
+                };
+            }
+        }
+        expect(error).toBeNull();
+        expect(JSON.stringify(result, null, 2)).toMatchSnapshot();
+    });
+});