Merge pull request #17 from atom/wl-quoted-keys

Relax quoted key restrictions
atom · Jan 12, 2018 · e70d63b · e70d63b
2 parents 3fbd87e + 6ee27c4
commit e70d63b
Show file tree

Hide file tree

Showing 2 changed files with 55 additions and 7 deletions.
diff --git a/grammars/toml.cson b/grammars/toml.cson
@@ -29,16 +29,22 @@
       '2': 'name': 'keyword.operator.assignment.toml'
   }
   {
-    'match': '((")([\\w\\s-]*)("))\\s*(=)' # We use \\w here because quoted keys allow for a much wider variety.
+    'match': '((")(.*)("))\\s*(=)' # This one is .* because " can be escaped
     'captures':
       '1': 'name': 'string.quoted.double.toml'
       '2': 'name': 'punctuation.definition.string.begin.toml'
-      '3': 'name': 'variable.other.key.toml'
+      '3':
+        'name': 'variable.other.key.toml'
+        'patterns': [
+          {
+            'include': '#string-escapes'
+          }
+        ]
       '4': 'name': 'punctuation.definition.string.end.toml'
       '5': 'name': 'keyword.operator.assignment.toml'
   }
   {
-    'match': "((')([\\w\\s-]*)('))\\s*(=)" # We use \\w here because quoted keys allow for a much wider variety.
+    'match': "((')([^']*)('))\\s*(=)"
     'captures':
       '1': 'name': 'string.quoted.single.toml'
       '2': 'name': 'punctuation.definition.string.begin.toml'
@@ -55,8 +61,13 @@
       '0': 'name': 'punctuation.definition.string.end.toml'
     'name': 'string.quoted.double.block.toml'
     'patterns': [
-      'match': '\\\\[btnfr"\\\\]|\\\\u[A-Fa-f0-9]{4}|\\\\U[A-Fa-f0-9]{8}|\\\\$'
-      'name': 'constant.character.escape.toml'
+      {
+        'include': '#string-escapes'
+      }
+      {
+        'match': '\\\\$'
+        'name': 'constant.character.escape.toml'
+      }
     ]
   }
   {
@@ -77,8 +88,9 @@
       '0': 'name': 'punctuation.definition.string.end.toml'
     'name': 'string.quoted.double.toml'
     'patterns': [
-      'match': '\\\\[btnfr"\\\\]|\\\\u[A-Fa-f0-9]{4}|\\\\U[A-Fa-f0-9]{8}'
-      'name': 'constant.character.escape.toml'
+      {
+        'include': '#string-escapes'
+      }
     ]
   }
   {
@@ -111,3 +123,7 @@
     'name': 'constant.numeric.toml'
   }
 ]
+'repository':
+  'string-escapes':
+    'match': '\\\\[btnfr"\\\\]|\\\\u[A-Fa-f0-9]{4}|\\\\U[A-Fa-f0-9]{8}'
+    'name': 'constant.character.escape.toml'
diff --git a/spec/toml-spec.coffee b/spec/toml-spec.coffee
@@ -163,6 +163,13 @@ describe "TOML grammar", ->
     expect(tokens[3]).toEqual value: " ", scopes: ["source.toml"]
     expect(tokens[4]).toEqual value: "=", scopes: ["source.toml", "keyword.operator.assignment.toml"]
 
+    {tokens} = grammar.tokenizeLine("'key with colons:' =")
+    expect(tokens[0]).toEqual value: "'", scopes: ["source.toml", "string.quoted.single.toml", "punctuation.definition.string.begin.toml"]
+    expect(tokens[1]).toEqual value: "key with colons:", scopes: ["source.toml", "string.quoted.single.toml", "variable.other.key.toml"]
+    expect(tokens[2]).toEqual value: "'", scopes: ["source.toml", "string.quoted.single.toml", "punctuation.definition.string.end.toml"]
+    expect(tokens[3]).toEqual value: " ", scopes: ["source.toml"]
+    expect(tokens[4]).toEqual value: "=", scopes: ["source.toml", "keyword.operator.assignment.toml"]
+
     {tokens} = grammar.tokenizeLine("'' =")
     expect(tokens[0]).toEqual value: "'", scopes: ["source.toml", "string.quoted.single.toml", "punctuation.definition.string.begin.toml"]
     expect(tokens[1]).toEqual value: "'", scopes: ["source.toml", "string.quoted.single.toml", "punctuation.definition.string.end.toml"]
@@ -190,6 +197,31 @@ describe "TOML grammar", ->
     expect(tokens[3]).toEqual value: " ", scopes: ["source.toml"]
     expect(tokens[4]).toEqual value: "=", scopes: ["source.toml", "keyword.operator.assignment.toml"]
 
+    {tokens} = grammar.tokenizeLine('"key with colons:" =')
+    expect(tokens[0]).toEqual value: '"', scopes: ["source.toml", "string.quoted.double.toml", "punctuation.definition.string.begin.toml"]
+    expect(tokens[1]).toEqual value: "key with colons:", scopes: ["source.toml", "string.quoted.double.toml", "variable.other.key.toml"]
+    expect(tokens[2]).toEqual value: '"', scopes: ["source.toml", "string.quoted.double.toml", "punctuation.definition.string.end.toml"]
+    expect(tokens[3]).toEqual value: " ", scopes: ["source.toml"]
+    expect(tokens[4]).toEqual value: "=", scopes: ["source.toml", "keyword.operator.assignment.toml"]
+
+    {tokens} = grammar.tokenizeLine('"key wi\\th escapes" =')
+    expect(tokens[0]).toEqual value: '"', scopes: ["source.toml", "string.quoted.double.toml", "punctuation.definition.string.begin.toml"]
+    expect(tokens[1]).toEqual value: "key wi", scopes: ["source.toml", "string.quoted.double.toml", "variable.other.key.toml"]
+    expect(tokens[2]).toEqual value: "\\t", scopes: ["source.toml", "string.quoted.double.toml", "variable.other.key.toml", "constant.character.escape.toml"]
+    expect(tokens[3]).toEqual value: "h escapes", scopes: ["source.toml", "string.quoted.double.toml", "variable.other.key.toml"]
+    expect(tokens[4]).toEqual value: '"', scopes: ["source.toml", "string.quoted.double.toml", "punctuation.definition.string.end.toml"]
+    expect(tokens[5]).toEqual value: " ", scopes: ["source.toml"]
+    expect(tokens[6]).toEqual value: "=", scopes: ["source.toml", "keyword.operator.assignment.toml"]
+
+    {tokens} = grammar.tokenizeLine('"key with \\" quote" =')
+    expect(tokens[0]).toEqual value: '"', scopes: ["source.toml", "string.quoted.double.toml", "punctuation.definition.string.begin.toml"]
+    expect(tokens[1]).toEqual value: "key with ", scopes: ["source.toml", "string.quoted.double.toml", "variable.other.key.toml"]
+    expect(tokens[2]).toEqual value: '\\"', scopes: ["source.toml", "string.quoted.double.toml", "variable.other.key.toml", "constant.character.escape.toml"]
+    expect(tokens[3]).toEqual value: " quote", scopes: ["source.toml", "string.quoted.double.toml", "variable.other.key.toml"]
+    expect(tokens[4]).toEqual value: '"', scopes: ["source.toml", "string.quoted.double.toml", "punctuation.definition.string.end.toml"]
+    expect(tokens[5]).toEqual value: " ", scopes: ["source.toml"]
+    expect(tokens[6]).toEqual value: "=", scopes: ["source.toml", "keyword.operator.assignment.toml"]
+
     {tokens} = grammar.tokenizeLine('"" =')
     expect(tokens[0]).toEqual value: '"', scopes: ["source.toml", "string.quoted.double.toml", "punctuation.definition.string.begin.toml"]
     expect(tokens[1]).toEqual value: '"', scopes: ["source.toml", "string.quoted.double.toml", "punctuation.definition.string.end.toml"]