Skip to content
This repository has been archived by the owner on Mar 3, 2023. It is now read-only.

Commit

Permalink
Merge pull request #17339 from atom/mb-async-parsing
Browse files Browse the repository at this point in the history 
Perform parsing off the main thread when Tree-sitter is enabled
  • Loading branch information
Max Brunsfeld authored May 23, 2018
2 parents 6a5acd9 + 53dfa83 commit a78d682
Show file tree
Hide file tree
Showing 4 changed files with 276 additions and 145 deletions.
4 changes: 2 additions & 2 deletions package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -71,8 +71,8 @@
"service-hub": "^0.7.4",
"sinon": "1.17.4",
"temp": "^0.8.3",
"text-buffer": "13.14.2",
"tree-sitter": "^0.11.2",
"text-buffer": "13.14.3",
"tree-sitter": "0.12.6",
"typescript-simple": "1.0.0",
"underscore-plus": "^1.6.6",
"winreg": "^1.2.1",
Expand Down
137 changes: 109 additions & 28 deletions spec/tree-sitter-language-mode-spec.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ describe('TreeSitterLanguageMode', () => {
})

describe('highlighting', () => {
it('applies the most specific scope mapping to each node in the syntax tree', () => {
it('applies the most specific scope mapping to each node in the syntax tree', async () => {
const grammar = new TreeSitterGrammar(atom.grammars, jsGrammarPath, {
parser: 'tree-sitter-javascript',
scopes: {
Expand All @@ -31,8 +31,11 @@ describe('TreeSitterLanguageMode', () => {
}
})

buffer.setLanguageMode(new TreeSitterLanguageMode({buffer, grammar}))
const languageMode = new TreeSitterLanguageMode({buffer, grammar})
buffer.setLanguageMode(languageMode)
buffer.setText('aa.bbb = cc(d.eee());')
await languageMode.reparsePromise

expectTokensToEqual(editor, [[
{text: 'aa.', scopes: ['source']},
{text: 'bbb', scopes: ['source', 'property']},
Expand All @@ -44,7 +47,7 @@ describe('TreeSitterLanguageMode', () => {
]])
})

it('can start or end multiple scopes at the same position', () => {
it('can start or end multiple scopes at the same position', async () => {
const grammar = new TreeSitterGrammar(atom.grammars, jsGrammarPath, {
parser: 'tree-sitter-javascript',
scopes: {
Expand All @@ -57,8 +60,11 @@ describe('TreeSitterLanguageMode', () => {
}
})

buffer.setLanguageMode(new TreeSitterLanguageMode({buffer, grammar}))
const languageMode = new TreeSitterLanguageMode({buffer, grammar})
buffer.setLanguageMode(languageMode)
buffer.setText('a = bb.ccc();')
await languageMode.reparsePromise

expectTokensToEqual(editor, [[
{text: 'a', scopes: ['source', 'variable']},
{text: ' = ', scopes: ['source']},
Expand All @@ -70,7 +76,7 @@ describe('TreeSitterLanguageMode', () => {
]])
})

it('can resume highlighting on a line that starts with whitespace', () => {
it('can resume highlighting on a line that starts with whitespace', async () => {
const grammar = new TreeSitterGrammar(atom.grammars, jsGrammarPath, {
parser: 'tree-sitter-javascript',
scopes: {
Expand All @@ -80,8 +86,11 @@ describe('TreeSitterLanguageMode', () => {
}
})

buffer.setLanguageMode(new TreeSitterLanguageMode({buffer, grammar}))
const languageMode = new TreeSitterLanguageMode({buffer, grammar})
buffer.setLanguageMode(languageMode)
buffer.setText('a\n .b();')
await languageMode.reparsePromise

expectTokensToEqual(editor, [
[
{text: 'a', scopes: ['variable']},
Expand All @@ -95,7 +104,7 @@ describe('TreeSitterLanguageMode', () => {
])
})

it('correctly skips over tokens with zero size', () => {
it('correctly skips over tokens with zero size', async () => {
const grammar = new TreeSitterGrammar(atom.grammars, jsGrammarPath, {
parser: 'tree-sitter-c',
scopes: {
Expand All @@ -107,10 +116,11 @@ describe('TreeSitterLanguageMode', () => {
const languageMode = new TreeSitterLanguageMode({buffer, grammar})
buffer.setLanguageMode(languageMode)
buffer.setText('int main() {\n int a\n int b;\n}');
await languageMode.reparsePromise

editor.screenLineForScreenRow(0)
expect(
languageMode.document.rootNode.descendantForPosition(Point(1, 2), Point(1, 6)).toString()
languageMode.tree.rootNode.descendantForPosition(Point(1, 2), Point(1, 6)).toString()
).toBe('(declaration (primitive_type) (identifier) (MISSING))')

expectTokensToEqual(editor, [
Expand Down Expand Up @@ -139,7 +149,7 @@ describe('TreeSitterLanguageMode', () => {
])
})

it('updates lines\' highlighting when they are affected by distant changes', () => {
it('updates lines\' highlighting when they are affected by distant changes', async () => {
const grammar = new TreeSitterGrammar(atom.grammars, jsGrammarPath, {
parser: 'tree-sitter-javascript',
scopes: {
Expand All @@ -148,10 +158,12 @@ describe('TreeSitterLanguageMode', () => {
}
})

buffer.setLanguageMode(new TreeSitterLanguageMode({buffer, grammar}))
const languageMode = new TreeSitterLanguageMode({buffer, grammar})
buffer.setLanguageMode(languageMode)
buffer.setText('a(\nb,\nc\n')
await languageMode.reparsePromise

// missing closing paren
buffer.setText('a(\nb,\nc\n')
expectTokensToEqual(editor, [
[{text: 'a(', scopes: []}],
[{text: 'b,', scopes: []}],
Expand All @@ -160,6 +172,7 @@ describe('TreeSitterLanguageMode', () => {
])

buffer.append(')')
await languageMode.reparsePromise
expectTokensToEqual(editor, [
[
{text: 'a', scopes: ['function']},
Expand All @@ -171,7 +184,7 @@ describe('TreeSitterLanguageMode', () => {
])
})

it('handles edits after tokens that end between CR and LF characters (regression)', () => {
it('handles edits after tokens that end between CR and LF characters (regression)', async () => {
const grammar = new TreeSitterGrammar(atom.grammars, jsGrammarPath, {
parser: 'tree-sitter-javascript',
scopes: {
Expand All @@ -181,13 +194,14 @@ describe('TreeSitterLanguageMode', () => {
}
})

buffer.setLanguageMode(new TreeSitterLanguageMode({buffer, grammar}))

const languageMode = new TreeSitterLanguageMode({buffer, grammar})
buffer.setLanguageMode(languageMode)
buffer.setText([
'// abc',
'',
'a("b").c'
].join('\r\n'))
await languageMode.reparsePromise

expectTokensToEqual(editor, [
[{text: '// abc', scopes: ['comment']}],
Expand All @@ -201,6 +215,7 @@ describe('TreeSitterLanguageMode', () => {
])

buffer.insert([2, 0], ' ')
await languageMode.reparsePromise
expectTokensToEqual(editor, [
[{text: '// abc', scopes: ['comment']}],
[{text: '', scopes: []}],
Expand All @@ -213,14 +228,75 @@ describe('TreeSitterLanguageMode', () => {
]
])
})

describe('when the buffer changes during a parse', () => {
it('immediately parses again when the current parse completes', async () => {
const grammar = new TreeSitterGrammar(atom.grammars, jsGrammarPath, {
parser: 'tree-sitter-javascript',
scopes: {
'identifier': 'variable',
'call_expression > identifier': 'function',
'new_expression > call_expression > identifier': 'constructor'
}
})
const languageMode = new TreeSitterLanguageMode({buffer, grammar})
buffer.setLanguageMode(languageMode)

buffer.setText('abc;');
await languageMode.reparsePromise
expectTokensToEqual(editor, [
[
{text: 'abc', scopes: ['variable']},
{text: ';', scopes: []}
],
])

buffer.setTextInRange([[0, 3], [0, 3]], '()');
expectTokensToEqual(editor, [
[
{text: 'abc()', scopes: ['variable']},
{text: ';', scopes: []}
],
])

buffer.setTextInRange([[0, 0], [0, 0]], 'new ');
expectTokensToEqual(editor, [
[
{text: 'new ', scopes: []},
{text: 'abc()', scopes: ['variable']},
{text: ';', scopes: []}
],
])

await languageMode.reparsePromise
expect(languageMode.reparsePromise).not.toBeNull()
expectTokensToEqual(editor, [
[
{text: 'new ', scopes: []},
{text: 'abc', scopes: ['function']},
{text: '();', scopes: []}
],
])

await languageMode.reparsePromise
expect(languageMode.reparsePromise).toBeNull()
expectTokensToEqual(editor, [
[
{text: 'new ', scopes: []},
{text: 'abc', scopes: ['constructor']},
{text: '();', scopes: []}
],
])
})
})
})

describe('folding', () => {
beforeEach(() => {
editor.displayLayer.reset({foldCharacter: '…'})
})

it('can fold nodes that start and end with specified tokens', () => {
it('can fold nodes that start and end with specified tokens', async () => {
const grammar = new TreeSitterGrammar(atom.grammars, jsGrammarPath, {
parser: 'tree-sitter-javascript',
folds: [
Expand All @@ -235,7 +311,8 @@ describe('TreeSitterLanguageMode', () => {
]
})

buffer.setLanguageMode(new TreeSitterLanguageMode({buffer, grammar}))
const languageMode = new TreeSitterLanguageMode({buffer, grammar})
buffer.setLanguageMode(languageMode)
buffer.setText(dedent `
module.exports =
class A {
Expand All @@ -246,6 +323,7 @@ describe('TreeSitterLanguageMode', () => {
}
}
`)
await languageMode.reparsePromise

editor.screenLineForScreenRow(0)

Expand Down Expand Up @@ -275,7 +353,7 @@ describe('TreeSitterLanguageMode', () => {
`)
})

it('can fold nodes of specified types', () => {
it('can fold nodes of specified types', async () => {
const grammar = new TreeSitterGrammar(atom.grammars, jsGrammarPath, {
parser: 'tree-sitter-javascript',
folds: [
Expand All @@ -296,7 +374,8 @@ describe('TreeSitterLanguageMode', () => {
]
})

buffer.setLanguageMode(new TreeSitterLanguageMode({buffer, grammar}))
const languageMode = new TreeSitterLanguageMode({buffer, grammar})
buffer.setLanguageMode(languageMode)
buffer.setText(dedent `
const element1 = <Element
className='submit'
Expand All @@ -307,6 +386,7 @@ describe('TreeSitterLanguageMode', () => {
<span>world</span>
</Element>
`)
await languageMode.reparsePromise

editor.screenLineForScreenRow(0)

Expand Down Expand Up @@ -336,7 +416,7 @@ describe('TreeSitterLanguageMode', () => {
`)
})

it('can fold entire nodes when no start or end parameters are specified', () => {
it('can fold entire nodes when no start or end parameters are specified', async () => {
const grammar = new TreeSitterGrammar(atom.grammars, jsGrammarPath, {
parser: 'tree-sitter-javascript',
folds: [
Expand All @@ -346,7 +426,8 @@ describe('TreeSitterLanguageMode', () => {
]
})

buffer.setLanguageMode(new TreeSitterLanguageMode({buffer, grammar}))
const languageMode = new TreeSitterLanguageMode({buffer, grammar})
buffer.setLanguageMode(languageMode)
buffer.setText(dedent `
/**
* Important
Expand All @@ -355,6 +436,7 @@ describe('TreeSitterLanguageMode', () => {
Also important
*/
`)
await languageMode.reparsePromise

editor.screenLineForScreenRow(0)

Expand All @@ -379,7 +461,7 @@ describe('TreeSitterLanguageMode', () => {
`)
})

it('tries each folding strategy for a given node in the order specified', () => {
it('tries each folding strategy for a given node in the order specified', async () => {
const grammar = new TreeSitterGrammar(atom.grammars, cGrammarPath, {
parser: 'tree-sitter-c',
folds: [
Expand All @@ -405,8 +487,8 @@ describe('TreeSitterLanguageMode', () => {
]
})

buffer.setLanguageMode(new TreeSitterLanguageMode({buffer, grammar}))

const languageMode = new TreeSitterLanguageMode({buffer, grammar})
buffer.setLanguageMode(languageMode)
buffer.setText(dedent `
#ifndef FOO_H_
#define FOO_H_
Expand All @@ -430,6 +512,7 @@ describe('TreeSitterLanguageMode', () => {
#endif
`)
await languageMode.reparsePromise

editor.screenLineForScreenRow(0)

Expand Down Expand Up @@ -504,8 +587,6 @@ describe('TreeSitterLanguageMode', () => {
]
})

buffer.setLanguageMode(new TreeSitterLanguageMode({buffer, grammar}))

buffer.setText(dedent `
def ab():
print 'a'
Expand All @@ -515,6 +596,7 @@ describe('TreeSitterLanguageMode', () => {
print 'c'
print 'd'
`)
buffer.setLanguageMode(new TreeSitterLanguageMode({buffer, grammar}))

editor.screenLineForScreenRow(0)

Expand All @@ -537,9 +619,8 @@ describe('TreeSitterLanguageMode', () => {
parser: 'tree-sitter-javascript'
})

buffer.setLanguageMode(new TreeSitterLanguageMode({buffer, grammar}))

buffer.setText('foo({bar: baz});')
buffer.setLanguageMode(new TreeSitterLanguageMode({buffer, grammar}))

editor.screenLineForScreenRow(0)
expect(editor.scopeDescriptorForBufferPosition([0, 6]).getScopesArray()).toEqual([
Expand All @@ -562,13 +643,13 @@ describe('TreeSitterLanguageMode', () => {
scopes: {'program': 'source'}
})

buffer.setLanguageMode(new TreeSitterLanguageMode({buffer, grammar}))
buffer.setText(dedent `
function a (b, c, d) {
eee.f()
g()
}
`)
buffer.setLanguageMode(new TreeSitterLanguageMode({buffer, grammar}))

editor.screenLineForScreenRow(0)

Expand Down
Loading

0 comments on commit a78d682

Please sign in to comment.