Skip to content

Commit

Permalink
scopes -> scopeDescriptor
Browse files Browse the repository at this point in the history 
!!!
  • Loading branch information
@benogle
benogle committed Oct 13, 2014
1 parent 8cf36af commit c0091b4
Show file tree
Hide file tree
Showing 12 changed files with 116 additions and 119 deletions.
2 changes: 1 addition & 1 deletion spec/text-editor-component-spec.coffee
View file
Original file line number Diff line number Diff line change
Expand Up @@ -277,7 +277,7 @@ describe "TextEditorComponent", ->
expect(leafNodes[0].classList.contains('invisible-character')).toBe true
expect(leafNodes[leafNodes.length - 1].classList.contains('invisible-character')).toBe true

it "displays newlines as their own token outside of the other tokens' scopes", ->
it "displays newlines as their own token outside of the other tokens' scopeDescriptor", ->
editor.setText "var\n"
nextAnimationFrame()
expect(component.lineNodeForScreenRow(0).innerHTML).toBe "<span class=\"source js\"><span class=\"storage modifier js\">var</span></span><span class=\"invisible-character\">#{invisibles.eol}</span>"
Expand Down
14 changes: 7 additions & 7 deletions spec/text-editor-spec.coffee
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3650,10 +3650,10 @@ describe "TextEditor", ->
{tokens} = grammar.tokenizeLine("var i; // http://github.com")

expect(tokens[0].value).toBe "var"
expect(tokens[0].scopes).toEqual ["source.js", "storage.modifier.js"]
expect(tokens[0].scopeDescriptor).toEqual ["source.js", "storage.modifier.js"]

expect(tokens[6].value).toBe "http://github.com"
expect(tokens[6].scopes).toEqual ["source.js", "comment.line.double-slash.js", "markup.underline.link.http.hyperlink"]
expect(tokens[6].scopeDescriptor).toEqual ["source.js", "comment.line.double-slash.js", "markup.underline.link.http.hyperlink"]

describe "when the grammar is added", ->
it "retokenizes existing buffers that contain tokens that match the injection selector", ->
Expand All @@ -3665,15 +3665,15 @@ describe "TextEditor", ->

{tokens} = editor.tokenizedLineForScreenRow(0)
expect(tokens[1].value).toBe " http://github.com"
expect(tokens[1].scopes).toEqual ["source.js", "comment.line.double-slash.js"]
expect(tokens[1].scopeDescriptor).toEqual ["source.js", "comment.line.double-slash.js"]

waitsForPromise ->
atom.packages.activatePackage('language-hyperlink')

runs ->
{tokens} = editor.tokenizedLineForScreenRow(0)
expect(tokens[2].value).toBe "http://github.com"
expect(tokens[2].scopes).toEqual ["source.js", "comment.line.double-slash.js", "markup.underline.link.http.hyperlink"]
expect(tokens[2].scopeDescriptor).toEqual ["source.js", "comment.line.double-slash.js", "markup.underline.link.http.hyperlink"]

describe "when the grammar is updated", ->
it "retokenizes existing buffers that contain tokens that match the injection selector", ->
Expand All @@ -3685,23 +3685,23 @@ describe "TextEditor", ->

{tokens} = editor.tokenizedLineForScreenRow(0)
expect(tokens[1].value).toBe " SELECT * FROM OCTOCATS"
expect(tokens[1].scopes).toEqual ["source.js", "comment.line.double-slash.js"]
expect(tokens[1].scopeDescriptor).toEqual ["source.js", "comment.line.double-slash.js"]

waitsForPromise ->
atom.packages.activatePackage('package-with-injection-selector')

runs ->
{tokens} = editor.tokenizedLineForScreenRow(0)
expect(tokens[1].value).toBe " SELECT * FROM OCTOCATS"
expect(tokens[1].scopes).toEqual ["source.js", "comment.line.double-slash.js"]
expect(tokens[1].scopeDescriptor).toEqual ["source.js", "comment.line.double-slash.js"]

waitsForPromise ->
atom.packages.activatePackage('language-sql')

runs ->
{tokens} = editor.tokenizedLineForScreenRow(0)
expect(tokens[2].value).toBe "SELECT"
expect(tokens[2].scopes).toEqual ["source.js", "comment.line.double-slash.js", "keyword.other.DML.sql"]
expect(tokens[2].scopeDescriptor).toEqual ["source.js", "comment.line.double-slash.js", "keyword.other.DML.sql"]

describe ".normalizeTabsInBufferRange()", ->
it "normalizes tabs depending on the editor's soft tab/tab length settings", ->
Expand Down
84 changes: 42 additions & 42 deletions spec/tokenized-buffer-spec.coffee
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,12 +51,12 @@ describe "TokenizedBuffer", ->
it "initially creates un-tokenized screen lines, then tokenizes lines chunk at a time in the background", ->
line0 = tokenizedBuffer.tokenizedLineForRow(0)
expect(line0.tokens.length).toBe 1
expect(line0.tokens[0]).toEqual(value: line0.text, scopes: ['source.js'])
expect(line0.tokens[0]).toEqual(value: line0.text, scopeDescriptor: ['source.js'])

line11 = tokenizedBuffer.tokenizedLineForRow(11)
expect(line11.tokens.length).toBe 2
expect(line11.tokens[0]).toEqual(value: " ", scopes: ['source.js'], isAtomic: true)
expect(line11.tokens[1]).toEqual(value: "return sort(Array.apply(this, arguments));", scopes: ['source.js'])
expect(line11.tokens[0]).toEqual(value: " ", scopeDescriptor: ['source.js'], isAtomic: true)
expect(line11.tokens[1]).toEqual(value: "return sort(Array.apply(this, arguments));", scopeDescriptor: ['source.js'])

# background tokenization has not begun
expect(tokenizedBuffer.tokenizedLineForRow(0).ruleStack).toBeUndefined()
Expand Down Expand Up @@ -149,10 +149,10 @@ describe "TokenizedBuffer", ->
it "updates tokens to reflect the change", ->
buffer.setTextInRange([[0, 0], [2, 0]], "foo()\n7\n")

expect(tokenizedBuffer.tokenizedLineForRow(0).tokens[1]).toEqual(value: '(', scopes: ['source.js', 'meta.brace.round.js'])
expect(tokenizedBuffer.tokenizedLineForRow(1).tokens[0]).toEqual(value: '7', scopes: ['source.js', 'constant.numeric.js'])
expect(tokenizedBuffer.tokenizedLineForRow(0).tokens[1]).toEqual(value: '(', scopeDescriptor: ['source.js', 'meta.brace.round.js'])
expect(tokenizedBuffer.tokenizedLineForRow(1).tokens[0]).toEqual(value: '7', scopeDescriptor: ['source.js', 'constant.numeric.js'])
# line 2 is unchanged
expect(tokenizedBuffer.tokenizedLineForRow(2).tokens[2]).toEqual(value: 'if', scopes: ['source.js', 'keyword.control.js'])
expect(tokenizedBuffer.tokenizedLineForRow(2).tokens[2]).toEqual(value: 'if', scopeDescriptor: ['source.js', 'keyword.control.js'])

expect(changeHandler).toHaveBeenCalled()
[event] = changeHandler.argsForCall[0]
Expand All @@ -164,17 +164,17 @@ describe "TokenizedBuffer", ->
buffer.insert([5, 30], '/* */')
changeHandler.reset()
buffer.insert([2, 0], '/*')
expect(tokenizedBuffer.tokenizedLineForRow(3).tokens[0].scopes).toEqual ['source.js']
expect(tokenizedBuffer.tokenizedLineForRow(3).tokens[0].scopeDescriptor).toEqual ['source.js']
expect(changeHandler).toHaveBeenCalled()
[event] = changeHandler.argsForCall[0]
delete event.bufferChange
expect(event).toEqual(start: 2, end: 2, delta: 0)
changeHandler.reset()

advanceClock()
expect(tokenizedBuffer.tokenizedLineForRow(3).tokens[0].scopes).toEqual ['source.js', 'comment.block.js']
expect(tokenizedBuffer.tokenizedLineForRow(4).tokens[0].scopes).toEqual ['source.js', 'comment.block.js']
expect(tokenizedBuffer.tokenizedLineForRow(5).tokens[0].scopes).toEqual ['source.js', 'comment.block.js']
expect(tokenizedBuffer.tokenizedLineForRow(3).tokens[0].scopeDescriptor).toEqual ['source.js', 'comment.block.js']
expect(tokenizedBuffer.tokenizedLineForRow(4).tokens[0].scopeDescriptor).toEqual ['source.js', 'comment.block.js']
expect(tokenizedBuffer.tokenizedLineForRow(5).tokens[0].scopeDescriptor).toEqual ['source.js', 'comment.block.js']
expect(changeHandler).toHaveBeenCalled()
[event] = changeHandler.argsForCall[0]
delete event.bufferChange
Expand All @@ -185,23 +185,23 @@ describe "TokenizedBuffer", ->
buffer.insert([5, 0], '*/')

buffer.insert([1, 0], 'var ')
expect(tokenizedBuffer.tokenizedLineForRow(1).tokens[0].scopes).toEqual ['source.js', 'comment.block.js']
expect(tokenizedBuffer.tokenizedLineForRow(1).tokens[0].scopeDescriptor).toEqual ['source.js', 'comment.block.js']

describe "when lines are both updated and removed", ->
it "updates tokens to reflect the change", ->
buffer.setTextInRange([[1, 0], [3, 0]], "foo()")

# previous line 0 remains
expect(tokenizedBuffer.tokenizedLineForRow(0).tokens[0]).toEqual(value: 'var', scopes: ['source.js', 'storage.modifier.js'])
expect(tokenizedBuffer.tokenizedLineForRow(0).tokens[0]).toEqual(value: 'var', scopeDescriptor: ['source.js', 'storage.modifier.js'])

# previous line 3 should be combined with input to form line 1
expect(tokenizedBuffer.tokenizedLineForRow(1).tokens[0]).toEqual(value: 'foo', scopes: ['source.js'])
expect(tokenizedBuffer.tokenizedLineForRow(1).tokens[6]).toEqual(value: '=', scopes: ['source.js', 'keyword.operator.js'])
expect(tokenizedBuffer.tokenizedLineForRow(1).tokens[0]).toEqual(value: 'foo', scopeDescriptor: ['source.js'])
expect(tokenizedBuffer.tokenizedLineForRow(1).tokens[6]).toEqual(value: '=', scopeDescriptor: ['source.js', 'keyword.operator.js'])

# lines below deleted regions should be shifted upward
expect(tokenizedBuffer.tokenizedLineForRow(2).tokens[2]).toEqual(value: 'while', scopes: ['source.js', 'keyword.control.js'])
expect(tokenizedBuffer.tokenizedLineForRow(3).tokens[4]).toEqual(value: '=', scopes: ['source.js', 'keyword.operator.js'])
expect(tokenizedBuffer.tokenizedLineForRow(4).tokens[4]).toEqual(value: '<', scopes: ['source.js', 'keyword.operator.js'])
expect(tokenizedBuffer.tokenizedLineForRow(2).tokens[2]).toEqual(value: 'while', scopeDescriptor: ['source.js', 'keyword.control.js'])
expect(tokenizedBuffer.tokenizedLineForRow(3).tokens[4]).toEqual(value: '=', scopeDescriptor: ['source.js', 'keyword.operator.js'])
expect(tokenizedBuffer.tokenizedLineForRow(4).tokens[4]).toEqual(value: '<', scopeDescriptor: ['source.js', 'keyword.operator.js'])

expect(changeHandler).toHaveBeenCalled()
[event] = changeHandler.argsForCall[0]
Expand All @@ -214,17 +214,17 @@ describe "TokenizedBuffer", ->
changeHandler.reset()

buffer.setTextInRange([[2, 0], [3, 0]], '/*')
expect(tokenizedBuffer.tokenizedLineForRow(2).tokens[0].scopes).toEqual ['source.js', 'comment.block.js', 'punctuation.definition.comment.js']
expect(tokenizedBuffer.tokenizedLineForRow(3).tokens[0].scopes).toEqual ['source.js']
expect(tokenizedBuffer.tokenizedLineForRow(2).tokens[0].scopeDescriptor).toEqual ['source.js', 'comment.block.js', 'punctuation.definition.comment.js']
expect(tokenizedBuffer.tokenizedLineForRow(3).tokens[0].scopeDescriptor).toEqual ['source.js']
expect(changeHandler).toHaveBeenCalled()
[event] = changeHandler.argsForCall[0]
delete event.bufferChange
expect(event).toEqual(start: 2, end: 3, delta: -1)
changeHandler.reset()

advanceClock()
expect(tokenizedBuffer.tokenizedLineForRow(3).tokens[0].scopes).toEqual ['source.js', 'comment.block.js']
expect(tokenizedBuffer.tokenizedLineForRow(4).tokens[0].scopes).toEqual ['source.js', 'comment.block.js']
expect(tokenizedBuffer.tokenizedLineForRow(3).tokens[0].scopeDescriptor).toEqual ['source.js', 'comment.block.js']
expect(tokenizedBuffer.tokenizedLineForRow(4).tokens[0].scopeDescriptor).toEqual ['source.js', 'comment.block.js']
expect(changeHandler).toHaveBeenCalled()
[event] = changeHandler.argsForCall[0]
delete event.bufferChange
Expand All @@ -235,19 +235,19 @@ describe "TokenizedBuffer", ->
buffer.setTextInRange([[1, 0], [2, 0]], "foo()\nbar()\nbaz()\nquux()")

# previous line 0 remains
expect(tokenizedBuffer.tokenizedLineForRow(0).tokens[0]).toEqual( value: 'var', scopes: ['source.js', 'storage.modifier.js'])
expect(tokenizedBuffer.tokenizedLineForRow(0).tokens[0]).toEqual( value: 'var', scopeDescriptor: ['source.js', 'storage.modifier.js'])

# 3 new lines inserted
expect(tokenizedBuffer.tokenizedLineForRow(1).tokens[0]).toEqual(value: 'foo', scopes: ['source.js'])
expect(tokenizedBuffer.tokenizedLineForRow(2).tokens[0]).toEqual(value: 'bar', scopes: ['source.js'])
expect(tokenizedBuffer.tokenizedLineForRow(3).tokens[0]).toEqual(value: 'baz', scopes: ['source.js'])
expect(tokenizedBuffer.tokenizedLineForRow(1).tokens[0]).toEqual(value: 'foo', scopeDescriptor: ['source.js'])
expect(tokenizedBuffer.tokenizedLineForRow(2).tokens[0]).toEqual(value: 'bar', scopeDescriptor: ['source.js'])
expect(tokenizedBuffer.tokenizedLineForRow(3).tokens[0]).toEqual(value: 'baz', scopeDescriptor: ['source.js'])

# previous line 2 is joined with quux() on line 4
expect(tokenizedBuffer.tokenizedLineForRow(4).tokens[0]).toEqual(value: 'quux', scopes: ['source.js'])
expect(tokenizedBuffer.tokenizedLineForRow(4).tokens[4]).toEqual(value: 'if', scopes: ['source.js', 'keyword.control.js'])
expect(tokenizedBuffer.tokenizedLineForRow(4).tokens[0]).toEqual(value: 'quux', scopeDescriptor: ['source.js'])
expect(tokenizedBuffer.tokenizedLineForRow(4).tokens[4]).toEqual(value: 'if', scopeDescriptor: ['source.js', 'keyword.control.js'])

# previous line 3 is pushed down to become line 5
expect(tokenizedBuffer.tokenizedLineForRow(5).tokens[4]).toEqual(value: '=', scopes: ['source.js', 'keyword.operator.js'])
expect(tokenizedBuffer.tokenizedLineForRow(5).tokens[4]).toEqual(value: '=', scopeDescriptor: ['source.js', 'keyword.operator.js'])

expect(changeHandler).toHaveBeenCalled()
[event] = changeHandler.argsForCall[0]
Expand All @@ -264,17 +264,17 @@ describe "TokenizedBuffer", ->
[event] = changeHandler.argsForCall[0]
delete event.bufferChange
expect(event).toEqual(start: 2, end: 2, delta: 2)
expect(tokenizedBuffer.tokenizedLineForRow(2).tokens[0].scopes).toEqual ['source.js', 'comment.block.js', 'punctuation.definition.comment.js']
expect(tokenizedBuffer.tokenizedLineForRow(3).tokens[0].scopes).toEqual ['source.js', 'comment.block.js']
expect(tokenizedBuffer.tokenizedLineForRow(4).tokens[0].scopes).toEqual ['source.js', 'comment.block.js']
expect(tokenizedBuffer.tokenizedLineForRow(5).tokens[0].scopes).toEqual ['source.js']
expect(tokenizedBuffer.tokenizedLineForRow(2).tokens[0].scopeDescriptor).toEqual ['source.js', 'comment.block.js', 'punctuation.definition.comment.js']
expect(tokenizedBuffer.tokenizedLineForRow(3).tokens[0].scopeDescriptor).toEqual ['source.js', 'comment.block.js']
expect(tokenizedBuffer.tokenizedLineForRow(4).tokens[0].scopeDescriptor).toEqual ['source.js', 'comment.block.js']
expect(tokenizedBuffer.tokenizedLineForRow(5).tokens[0].scopeDescriptor).toEqual ['source.js']
changeHandler.reset()

advanceClock() # tokenize invalidated lines in background
expect(tokenizedBuffer.tokenizedLineForRow(5).tokens[0].scopes).toEqual ['source.js', 'comment.block.js']
expect(tokenizedBuffer.tokenizedLineForRow(6).tokens[0].scopes).toEqual ['source.js', 'comment.block.js']
expect(tokenizedBuffer.tokenizedLineForRow(7).tokens[0].scopes).toEqual ['source.js', 'comment.block.js']
expect(tokenizedBuffer.tokenizedLineForRow(8).tokens[0].scopes).not.toBe ['source.js', 'comment.block.js']
expect(tokenizedBuffer.tokenizedLineForRow(5).tokens[0].scopeDescriptor).toEqual ['source.js', 'comment.block.js']
expect(tokenizedBuffer.tokenizedLineForRow(6).tokens[0].scopeDescriptor).toEqual ['source.js', 'comment.block.js']
expect(tokenizedBuffer.tokenizedLineForRow(7).tokens[0].scopeDescriptor).toEqual ['source.js', 'comment.block.js']
expect(tokenizedBuffer.tokenizedLineForRow(8).tokens[0].scopeDescriptor).not.toBe ['source.js', 'comment.block.js']

expect(changeHandler).toHaveBeenCalled()
[event] = changeHandler.argsForCall[0]
Expand Down Expand Up @@ -343,7 +343,7 @@ describe "TokenizedBuffer", ->
expect(tokens[0].value).toBe "#"
expect(tokens[1].value).toBe " Econ 101"
expect(tokens[2].value).toBe tabAsSpaces
expect(tokens[2].scopes).toEqual tokens[1].scopes
expect(tokens[2].scopeDescriptor).toEqual tokens[1].scopeDescriptor
expect(tokens[2].isAtomic).toBeTruthy()
expect(tokens[3].value).toBe ""

Expand Down Expand Up @@ -526,15 +526,15 @@ describe "TokenizedBuffer", ->
fullyTokenize(tokenizedBuffer)

{tokens} = tokenizedBuffer.tokenizedLineForRow(0)
expect(tokens[0]).toEqual value: "<div class='name'>", scopes: ["text.html.ruby"]
expect(tokens[0]).toEqual value: "<div class='name'>", scopeDescriptor: ["text.html.ruby"]

waitsForPromise ->
atom.packages.activatePackage('language-html')

runs ->
fullyTokenize(tokenizedBuffer)
{tokens} = tokenizedBuffer.tokenizedLineForRow(0)
expect(tokens[0]).toEqual value: '<', scopes: ["text.html.ruby","meta.tag.block.any.html","punctuation.definition.tag.begin.html"]
expect(tokens[0]).toEqual value: '<', scopeDescriptor: ["text.html.ruby","meta.tag.block.any.html","punctuation.definition.tag.begin.html"]

describe ".tokenForPosition(position)", ->
afterEach ->
Expand All @@ -545,9 +545,9 @@ describe "TokenizedBuffer", ->
buffer = atom.project.bufferForPathSync('sample.js')
tokenizedBuffer = new TokenizedBuffer({buffer})
fullyTokenize(tokenizedBuffer)
expect(tokenizedBuffer.tokenForPosition([1,0]).scopes).toEqual ["source.js"]
expect(tokenizedBuffer.tokenForPosition([1,1]).scopes).toEqual ["source.js"]
expect(tokenizedBuffer.tokenForPosition([1,2]).scopes).toEqual ["source.js", "storage.modifier.js"]
expect(tokenizedBuffer.tokenForPosition([1,0]).scopeDescriptor).toEqual ["source.js"]
expect(tokenizedBuffer.tokenForPosition([1,1]).scopeDescriptor).toEqual ["source.js"]
expect(tokenizedBuffer.tokenForPosition([1,2]).scopeDescriptor).toEqual ["source.js", "storage.modifier.js"]

describe ".bufferRangeForScopeAtPosition(selector, position)", ->
beforeEach ->
Expand Down
8 changes: 4 additions & 4 deletions spec/tokenized-line-spec.coffee
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,16 +19,16 @@ describe "TokenizedLine", ->
expect(editor.tokenizedLineForScreenRow(2).isOnlyWhitespace()).toBe false

describe "::getScopeTree()", ->
it "returns a tree whose inner nodes are scopes and whose leaf nodes are tokens in those scopes", ->
it "returns a tree whose inner nodes are scopeDescriptor and whose leaf nodes are tokens in those scopeDescriptor", ->
[tokens, tokenIndex] = []

ensureValidScopeTree = (scopeTree, scopes=[]) ->
ensureValidScopeTree = (scopeTree, scopeDescriptor=[]) ->
if scopeTree.children?
for child in scopeTree.children
ensureValidScopeTree(child, scopes.concat([scopeTree.scope]))
ensureValidScopeTree(child, scopeDescriptor.concat([scopeTree.scope]))
else
expect(scopeTree).toBe tokens[tokenIndex++]
expect(scopes).toEqual scopeTree.scopes
expect(scopeDescriptor).toEqual scopeTree.scopeDescriptor

waitsForPromise ->
atom.project.open('coffee.coffee').then (o) -> editor = o
Expand Down
Loading

0 comments on commit c0091b4

Please sign in to comment.