allow slots from aliases, update spec, tests, add ci, badges

rodrigopivi · Sep 18, 2018 · 06a99b7 · 06a99b7
1 parent ad8c332
commit 06a99b7
Show file tree

Hide file tree

Showing 14 changed files with 2,691 additions and 2,827 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -0,0 +1,18 @@
+version: 2
+jobs:
+  build:
+    docker:
+      - image: circleci/node:8.11
+    working_directory: ~/repo
+    steps:
+      - checkout
+      - restore_cache:
+          keys:
+          - v1-dependencies-{{ checksum "package.json" }}
+          - v1-dependencies-
+      - run: npm install
+      - save_cache:
+          paths:
+            - node_modules
+          key: v1-dependencies-{{ checksum "package.json" }}
+      - run: npm run test
diff --git a/examples/citySearch_medium.chatito b/examples/citySearch_medium.chatito
@@ -46,7 +46,7 @@
 ~[city]
     city
 
-@[city]
+@[city]('entity': 'location')
     ~[newYork]
     ~[sanFrancisco]
     ~[atlanta]
diff --git a/package-lock.json b/package-lock.json
diff --git a/package.json b/package.json
@@ -1,6 +1,6 @@
 {
   "name": "chatito",
-  "version": "2.1.4",
+  "version": "2.1.5",
   "description": "Generate training datasets for NLU chatbots using a simple DSL",
   "bin": {
     "chatito": "./dist/bin.js"
@@ -67,45 +67,45 @@
     ]
   },
   "devDependencies": {
-    "@babel/core": "7.0.0-rc.1",
-    "@babel/plugin-proposal-class-properties": "7.0.0-rc.1",
-    "@babel/plugin-syntax-dynamic-import": "7.0.0-rc.1",
-    "@babel/plugin-syntax-import-meta": "7.0.0-rc.1",
-    "@babel/preset-env": "7.0.0-rc.1",
-    "@babel/preset-react": "7.0.0-rc.1",
-    "@babel/preset-typescript": "7.0.0-rc.1",
+    "@babel/core": "7.1.0",
+    "@babel/plugin-proposal-class-properties": "7.1.0",
+    "@babel/plugin-syntax-dynamic-import": "7.0.0",
+    "@babel/plugin-syntax-import-meta": "7.0.0",
+    "@babel/preset-env": "7.1.0",
+    "@babel/preset-react": "7.0.0",
+    "@babel/preset-typescript": "7.1.0",
     "@types/file-saver": "^1.3.0",
-    "@types/jest": "^23.3.1",
-    "@types/node": "^10.7.1",
-    "@types/react": "^16.4.11",
+    "@types/jest": "^23.3.2",
+    "@types/node": "^10.10.1",
+    "@types/react": "^16.4.14",
     "@types/react-dom": "^16.0.7",
     "@types/react-helmet": "^5.0.7",
-    "@types/react-router-dom": "^4.3.0",
-    "babel-loader": "^8.0.0-beta.2",
-    "babel-plugin-import": "^1.8.0",
-    "babel-plugin-styled-components": "^1.5.1",
+    "@types/react-router-dom": "^4.3.1",
+    "babel-loader": "^8.0.2",
+    "babel-plugin-import": "^1.9.1",
+    "babel-plugin-styled-components": "^1.7.1",
     "codeflask": "^1.2.1",
     "file-saver": "^1.3.8",
     "gatsby": "next",
-    "gatsby-link": "^1.6.46",
+    "gatsby-link": "^2.0.1",
     "gatsby-plugin-react-helmet": "next",
     "gatsby-plugin-styled-components": "next",
     "gatsby-plugin-typescript": "next",
-    "gh-pages": "^1.2.0",
-    "jest": "^23.5.0",
+    "gh-pages": "^2.0.0",
+    "jest": "^23.6.0",
     "pegjs": "^0.10.0",
     "prettier": "^1.14.2",
-    "react": "^16.4.2",
-    "react-dom": "^16.4.2",
+    "react": "^16.5.1",
+    "react-dom": "^16.5.1",
     "react-helmet": "^5.2.0",
     "react-json-view": "^1.19.1",
     "react-router-dom": "^4.3.1",
-    "styled-components": "^3.4.2",
+    "styled-components": "^3.4.9",
     "ts-jest": "^23.1.4",
     "ts-node": "^7.0.1",
     "tslint": "^5.11.0",
     "tslint-config-prettier": "^1.15.0",
-    "tslint-plugin-prettier": "^1.3.0",
-    "typescript": "^3.0.1"
+    "tslint-plugin-prettier": "^2.0.0",
+    "typescript": "^3.0.3"
   }
 }
diff --git a/parser/chatito.pegjs b/parser/chatito.pegjs
@@ -8,24 +8,18 @@ EntityOpt = "?"
 EntityBody = "[" value:EntityKeywordLiteral "]" { return value }
 EntityOptionalBody = "[" value:EntityKeywordLiteral opt:EntityOpt? "]"
     { return { value: value, opt: !!opt  }; }
-BasicKeywordLiterals = value:AnyTextWithAlias { return { value: value, type: "Text" }}
-// Entities (slot and aliases) allow any text except end of lines and alias definitions
-AnyTextWithAlias = v:(t:((!"\r\n")(!"\n")(!"~[") .) { return t.join(""); })+ { return v.join(""); }
-BasicValidInner = (OptionalAlias/BasicKeywordLiterals)+
-BasicInnerStatement =  Samedent s:BasicValidInner EOS { return s; }
-BasicInnerStatements =  BasicInnerStatement+
 
 // Intent
 EntityIntentDefinition = "%" value:EntityBody args:EntityArguments?
     { return { value: value, type: "IntentDefinition", args: args, location: location() } }
-// Intents allow any text except end of lines, alias and slot definitions
+// Intents allow any text except end of lines, alias and slot definitions (because they are parsed as another value)
 AnyTextWithSlotAndAlias = v:(t:((!"\r\n")(!"\n")(!"~[")(!"@[") .) { return t.join(""); })+ { return v.join(""); }
-IntentKeywordLiterals = value:AnyTextWithSlotAndAlias { return { value: value, type: "Text" }}
-IntentValidInner = (OptionalSlot/OptionalAlias/IntentKeywordLiterals)+
-IntentInnerStatements =  IntentInnerStatement+
-IntentInnerStatement =  Samedent s:IntentValidInner EOS { return s; }
+IntentAndSlotKeywordLiterals = value:AnyTextWithSlotAndAlias { return { value: value, type: "Text" }}
+IntentAndSlotValidInner = (OptionalSlot/OptionalAlias/IntentAndSlotKeywordLiterals)+
+IntentAndSlotInnerStatements =  IntentAndSlotInnerStatement+
+IntentAndSlotInnerStatement =  Samedent s:IntentAndSlotValidInner EOS { return s; }
 IntentDefinition = EOL? o:EntityIntentDefinition EOL
-    Indent s:IntentInnerStatements Dedent
+    Indent s:IntentAndSlotInnerStatements Dedent
     { return { type: o.type, key: o.value, args: o.args, location: o.location, inner: s } }
 
 // Slot
@@ -37,14 +31,21 @@ SlotOptionalBody = "[" value:SlotKeywordLiteral variation:SlotVariationDefinitio
     { return { value: value, opt: !!opt, variation: variation }; }
 OptionalSlot = "@" op:SlotOptionalBody
     { return { value: op.value, type: "Slot", opt: op.opt, location: location(), variation: op.variation } }
+// Slots allow any text except end of lines and alias definitions (because they are parsed as another value)
+AnyTextWithAlias = v:(t:((!"\r\n")(!"\n")(!"~[") .) { return t.join(""); })+ { return v.join(""); }
+SlotKeywordLiterals = value:AnyTextWithAlias { return { value: value, type: "Text" }}
+SlotValidInner = (OptionalAlias/SlotKeywordLiterals)+
+SlotInnerStatement =  Samedent s:SlotValidInner EOS { return s; }
+SlotInnerStatements =  SlotInnerStatement+
 SlotDefinition = EOL? o:EntitySlotDefinition EOL
-    Indent s:BasicInnerStatements Dedent
+    Indent s:SlotInnerStatements Dedent
     { return { type: o.type, key: o.value, args: o.args, location: o.location, inner: s, variation: o.variation } }
 
 // Alias
 EntityAliasDefinition = "~" value:EntityBody { return { value: value, type: "AliasDefinition", location: location() } }
 OptionalAlias = "~" op:EntityOptionalBody { return { value: op.value, type: "Alias", opt: op.opt } }
-AliasDefinition = EOL? o:EntityAliasDefinition EOL Indent s:BasicInnerStatements Dedent
+AliasDefinition = EOL? o:EntityAliasDefinition EOL
+    Indent s:IntentAndSlotInnerStatements Dedent
     { return { type: o.type, key: o.value, location: o.location, inner: s } }
 
 // ============= Identation =============

diff --git a/readme.md b/readme.md
@@ -1,9 +1,22 @@
 # Chatito
 
+[![npm version](https://badge.fury.io/js/chatito.svg)](https://badge.fury.io/js/chatito)
+![CircleCI (all branches)](https://img.shields.io/circleci/project/github/rodrigopivi/Chatito.svg)
+![npm](https://img.shields.io/npm/dm/chatito.svg)
+![GitHub](https://img.shields.io/github/license/rodrigopivi/Chatito.svg)
+
+
 [![Alt text](screenshot.jpg?raw=true "Screenshot of online IDE")](https://rodrigopivi.github.io/Chatito/)
 
 [Try the online IDE!](https://rodrigopivi.github.io/Chatito/)
 
+## Donate
+
+[![Alt text](https://c5.patreon.com/external/logo/become_a_patron_button.png "Become a Patron!")](https://www.patreon.com/bePatron?u=13643440)
+
+Designing and maintaining chatito takes time and effort, if it was usefull for you, please consider making a donation and share the abundance! :)
+
+## Overview
 Chatito helps you generate datasets for training and validating chatbot models using a minimalistic DSL.
 
 If you are building chatbots using commercial models, open source frameworks or writing your own natural language processing model, you need training examples. Chatito is here to help you.
@@ -99,8 +112,5 @@ npx chatito <pathToFileOrDirectory> --format=<format> --formatOptions=<formatOpt
  - `<formatOptions>` Optional. Path to a .json file that each adapter optionally can use
  - `<outputPath>` Optional. The directory where to save the generated dataset. Uses the current directory as default.
 
-### Donate
-Designing and maintaining chatito takes time and effort, if it was usefull for you, please consider making a donation and share the abundance! :)
-
 ### Author and maintainer
 Rodrigo Pimentel
diff --git a/spec.md b/spec.md
@@ -171,7 +171,7 @@ When an alias is referenced inside a slot definition, and it is the only token o
 
 Alias definitions are not allowed to declare entity arguments.
 
-Nesting entities: Sentences defined inside aliases can reference other aliases but preventing recursive loops
+Nesting entities: Sentences defined inside aliases can reference slots and other aliases but preventing recursive loops
 
 ## 3 - Data Generation
 

diff --git a/src/adapters/rasa.ts b/src/adapters/rasa.ts
@@ -33,42 +33,39 @@ export async function adapter(dsl: string, formatOptions?: any) {
             common_examples: []
         }
     };
-    const testing: IRasaTestingDataset = {};
+    const testing = { rasa_nlu_data: { common_examples: [] as IRasaExample[] } };
     const synonyms: { [key: string]: Set<string> } = {};
     if (formatOptions) {
         utils.mergeDeep(training, formatOptions);
     }
     const utteranceWriter = (utterance: ISentenceTokens[], intentKey: string, isTrainingExample: boolean) => {
-        if (isTrainingExample) {
-            const example = utterance.reduce(
-                (acc, next) => {
-                    if (next.type === 'Slot' && next.slot) {
-                        if (next.synonym) {
-                            if (!synonyms[next.synonym]) {
-                                synonyms[next.synonym] = new Set();
-                            }
-                            if (next.synonym !== next.value) {
-                                synonyms[next.synonym].add(next.value);
-                            }
+        const example = utterance.reduce(
+            (acc, next) => {
+                if (next.type === 'Slot' && next.slot) {
+                    if (next.synonym) {
+                        if (!synonyms[next.synonym]) {
+                            synonyms[next.synonym] = new Set();
+                        }
+                        if (next.synonym !== next.value) {
+                            synonyms[next.synonym].add(next.value);
                         }
-                        acc.entities.push({
-                            end: acc.text.length + next.value.length,
-                            entity: next.slot,
-                            start: acc.text.length,
-                            value: next.value
-                        });
                     }
-                    acc.text += next.value;
-                    return acc;
-                },
-                { text: '', intent: intentKey, entities: [] } as IRasaExample
-            );
+                    acc.entities.push({
+                        end: acc.text.length + next.value.length,
+                        entity: next.slot,
+                        start: acc.text.length,
+                        value: next.value
+                    });
+                }
+                acc.text += next.value;
+                return acc;
+            },
+            { text: '', intent: intentKey, entities: [] } as IRasaExample
+        );
+        if (isTrainingExample) {
             training.rasa_nlu_data.common_examples.push(example);
         } else {
-            if (!testing[intentKey]) {
-                testing[intentKey] = [];
-            }
-            testing[intentKey].push(utterance);
+            testing.rasa_nlu_data.common_examples.push(example);
         }
     };
     await gen.datasetFromString(dsl, utteranceWriter);

diff --git a/src/adapters/snips.ts b/src/adapters/snips.ts
@@ -60,17 +60,17 @@ export async function adapter(dsl: string, formatOptions?: any) {
                     if (!ret.entity) {
                         ret.entity = u.slot;
                         entities.add(u.slot);
-                        if (u.synonym) {
-                            if (!synonymsForSlots[u.slot]) {
-                                synonymsForSlots[u.slot] = {};
-                            }
-                            const synonyms = synonymsForSlots[u.slot];
-                            if (!synonyms[u.synonym]) {
-                                synonyms[u.synonym] = new Set();
-                            }
-                            if (u.synonym !== u.value) {
-                                synonyms[u.synonym].add(u.value);
-                            }
+                    }
+                    if (u.synonym && ret.entity) {
+                        if (!synonymsForSlots[ret.entity]) {
+                            synonymsForSlots[ret.entity] = {};
+                        }
+                        const synonyms = synonymsForSlots[ret.entity];
+                        if (!synonyms[u.synonym]) {
+                            synonyms[u.synonym] = new Set();
+                        }
+                        if (u.synonym !== u.value) {
+                            synonyms[u.synonym].add(u.value);
                         }
                     }
                 }

diff --git a/src/bin.ts b/src/bin.ts
@@ -60,7 +60,7 @@ const adapterAccumulator = (format: 'default' | 'rasa' | 'snips', formatOptions?
                 const testingJsonFilePath = path.resolve(outputPath, `${format}_dataset_testing.json`);
                 fs.writeFileSync(testingJsonFilePath, JSON.stringify(testingDataset));
                 // tslint:disable-next-line:no-console
-                console.log(`Saved testing dataset: ./${format}_dataset_training.json`);
+                console.log(`Saved testing dataset: ./${format}_dataset_testing.json`);
             }
         }
     };