v2.1.0: Added Emoji support

vtrushin · Dec 23, 2018 · b237594 · b237594
1 parent 18c727a
commit b237594
Show file tree

Hide file tree

Showing 14 changed files with 69 additions and 18 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,3 +1,6 @@
+## v2.1.0
+- Added Emoji support
+
 ## v2.0.7
 - Decreased minimal NodeJS version to 4 (minimal NodeJS version 6 will be in the next major version of "json-to-ast")
 

diff --git a/lib/error.js b/lib/error.js
@@ -1,5 +1,5 @@
 import codeErrorFragment from 'code-error-fragment';
-import createError from './create-error';
+import createError from './utils/create-error';
 
 export default (message, input, source, line, column) => {
 	throw createError({

diff --git a/lib/parseErrorTypes.js → lib/parse-error-types.js b/lib/parseErrorTypes.js → lib/parse-error-types.js
diff --git a/lib/parse.js b/lib/parse.js
@@ -1,7 +1,8 @@
+import substring from './utils/substring';
 import location from './location';
 import error from './error';
-import parseErrorTypes from './parseErrorTypes';
-import { tokenize, tokenTypes } from './tokenize';
+import parseErrorTypes from './parse-error-types';
+import tokenize, { tokenTypes } from './tokenize';
 
 const objectStates = {
 	_START_: 0,
@@ -63,8 +64,7 @@ const escapes = {
 
 const passEscapes = ['"', '\\', '/'];
 
-/** @param {string} string */
-function parseString(string) {
+function parseString(/** string */string) {
 	let result = '';
 
 	for (let i = 0; i < string.length; i ++) {
@@ -164,7 +164,7 @@ function parseObject(input, tokenList, index, settings) {
 				} else {
 					error(
 						parseErrorTypes.unexpectedToken(
-							input.substring(token.loc.start.offset, token.loc.end.offset),
+							substring(input, token.loc.start.offset, token.loc.end.offset),
 							settings.source,
 							token.loc.start.line,
 							token.loc.start.column
@@ -187,7 +187,7 @@ function parseObject(input, tokenList, index, settings) {
 				} else {
 					error(
 						parseErrorTypes.unexpectedToken(
-							input.substring(token.loc.start.offset, token.loc.end.offset),
+							substring(input, token.loc.start.offset, token.loc.end.offset),
 							settings.source,
 							token.loc.start.line,
 							token.loc.start.column
@@ -247,7 +247,7 @@ function parseProperty(input, tokenList, index, settings) {
 				} else {
 					error(
 						parseErrorTypes.unexpectedToken(
-							input.substring(token.loc.start.offset, token.loc.end.offset),
+							substring(input, token.loc.start.offset, token.loc.end.offset),
 							settings.source,
 							token.loc.start.line,
 							token.loc.start.column
@@ -359,7 +359,7 @@ function parseArray(input, tokenList, index, settings) {
 				} else {
 					error(
 						parseErrorTypes.unexpectedToken(
-							input.substring(token.loc.start.offset, token.loc.end.offset),
+							substring(input, token.loc.start.offset, token.loc.end.offset),
 							settings.source,
 							token.loc.start.line,
 							token.loc.start.column
@@ -446,7 +446,7 @@ function parseValue(input, tokenList, index, settings) {
 	} else {
 		error(
 			parseErrorTypes.unexpectedToken(
-				input.substring(token.loc.start.offset, token.loc.end.offset),
+				substring(input, token.loc.start.offset, token.loc.end.offset),
 				settings.source,
 				token.loc.start.line,
 				token.loc.start.column
@@ -478,7 +478,7 @@ export default (input, settings) => {
 
 	error(
 		parseErrorTypes.unexpectedToken(
-			input.substring(token.loc.start.offset, token.loc.end.offset),
+			substring(input, token.loc.start.offset, token.loc.end.offset),
 			settings.source,
 			token.loc.start.line,
 			token.loc.start.column

diff --git a/lib/tokenizeErrorTypes.js → lib/tokenize-error-types.js b/lib/tokenizeErrorTypes.js → lib/tokenize-error-types.js
diff --git a/lib/tokenize.js b/lib/tokenize.js
@@ -1,6 +1,7 @@
+import substring from './utils/substring';
 import location from './location';
 import error from './error';
-import tokenizeErrorTypes from './tokenizeErrorTypes';
+import tokenizeErrorTypes from './tokenize-error-types';
 
 export const tokenTypes = {
 	LEFT_BRACE: 0,		// {
@@ -329,7 +330,7 @@ function parseNumber(input, index, line, column) {
 	return null;
 }
 
-export function tokenize(input, settings) {
+const tokenize = (input, settings) => {
 	let line = 1;
 	let column = 1;
 	let index = 0;
@@ -375,7 +376,12 @@ export function tokenize(input, settings) {
 
 		} else {
 			error(
-				tokenizeErrorTypes.unexpectedSymbol(input.charAt(index), settings.source, line, column),
+				tokenizeErrorTypes.unexpectedSymbol(
+					substring(input, index, index + 1),
+					settings.source,
+					line,
+					column
+				),
 				input,
 				settings.source,
 				line,
@@ -386,4 +392,6 @@ export function tokenize(input, settings) {
 	}
 
 	return tokens;
-}
+};
+
+export default tokenize;
diff --git a/lib/create-error.js → lib/utils/create-error.js b/lib/create-error.js → lib/utils/create-error.js
diff --git a/lib/utils/substring.js b/lib/utils/substring.js
@@ -0,0 +1,25 @@
+import GraphemeSplitter from 'grapheme-splitter';
+
+const splitter = new GraphemeSplitter();
+
+const substring = (str, start, end) => {
+	const iterator = splitter.iterateGraphemes(str);
+
+	let value = '';
+
+	for (let pos = 0; pos < end; pos ++) {
+		const next = iterator.next();
+
+		if (pos >= start) {
+			value += next.value;
+		}
+
+		if (next.done) {
+			break;
+		}
+	}
+
+	return value;
+};
+
+export default substring;
diff --git a/package.json b/package.json
@@ -1,6 +1,6 @@
 {
   "name": "json-to-ast",
-  "version": "2.0.7",
+  "version": "2.1.0",
   "author": "Vlad Trushin",
   "description": "JSON AST parser",
   "homepage": "https://github.com/vtrushin/json-to-ast",
@@ -43,13 +43,16 @@
     "babel-plugin-transform-object-assign": "^6.22.0",
     "babel-preset-es2015": "^6.24.1",
     "babel-preset-stage-3": "^6.24.1",
-    "code-error-fragment": "0.0.230",
     "coveralls": "^3.0.2",
     "mocha": "5.2.0",
     "rollup": "^0.50.0",
     "rollup-plugin-babel": "^3.0.2",
     "rollup-plugin-commonjs": "^8.2.4",
     "rollup-plugin-node-resolve": "^3.0.0"
   },
+  "dependencies": {
+    "code-error-fragment": "0.0.230",
+    "grapheme-splitter": "^1.0.4"
+  },
   "license": "MIT"
 }
diff --git a/rollup.config.js b/rollup.config.js
@@ -17,7 +17,7 @@ export default {
 		babel()
 	],
 	watch: {
-		include: 'lib/*.js',
+		include: 'lib/**/*.js',
 		exclude: 'node_modules/**'
 	}
 }
diff --git a/test/fixtures/invalid/emoji-combined.js b/test/fixtures/invalid/emoji-combined.js
@@ -0,0 +1,5 @@
+module.exports = {
+	error: {
+		message: 'Unexpected symbol <👨‍👨‍👧‍👧> at 1:1'
+	}
+};
diff --git a/test/fixtures/invalid/emoji-combined.json b/test/fixtures/invalid/emoji-combined.json
@@ -0,0 +1 @@
+👨‍👨‍👧‍👧
diff --git a/test/fixtures/invalid/emoji.js b/test/fixtures/invalid/emoji.js
@@ -0,0 +1,5 @@
+module.exports = {
+	error: {
+		message: 'Unexpected symbol <💩> at 1:1'
+	}
+};
diff --git a/test/fixtures/invalid/emoji.json b/test/fixtures/invalid/emoji.json
@@ -0,0 +1 @@
+💩