Skip to content
This repository has been archived by the owner on Mar 3, 2023. It is now read-only.

Perform parsing off the main thread when Tree-sitter is enabled #17339

Merged
merged 7 commits into from
May 23, 2018
Merged

Perform parsing off the main thread when Tree-sitter is enabled #17339

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
a66120a
:arrow_up: text-buffer, tree-sitter
maxbrunsfeld May 15, 2018
aced30d
Start work on async parsing
maxbrunsfeld May 16, 2018
f6d2d57
Reparse again if there were changes since the last parse started
maxbrunsfeld May 22, 2018
3548abe
Fix bug w/ empty node handling, comment TreeSitterHighlightIterator
maxbrunsfeld May 22, 2018
d4d57c2
:racehorse: Parse asynchronously when opening buffers
maxbrunsfeld May 23, 2018
7a26674
Rename out-of-date property: layer -> languageMode
maxbrunsfeld May 23, 2018
53dfa83
Allow some synchronous parsing to avoid unnecessary re-renders
maxbrunsfeld May 23, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Start work on async parsing
  • Loading branch information
@maxbrunsfeld
maxbrunsfeld committed May 23, 2018
commit aced30da1f12eece514ba70cc22d63bebbb0da2a
2 changes: 1 addition & 1 deletion package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,7 @@
"sinon": "1.17.4",
"temp": "^0.8.3",
"text-buffer": "13.14.3",
"tree-sitter": "0.12.1-0",
"tree-sitter": "0.12.1-1",
"typescript-simple": "1.0.0",
"underscore-plus": "^1.6.6",
"winreg": "^1.2.1",
Expand Down
76 changes: 48 additions & 28 deletions spec/tree-sitter-language-mode-spec.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ describe('TreeSitterLanguageMode', () => {
})

describe('highlighting', () => {
it('applies the most specific scope mapping to each node in the syntax tree', () => {
it('applies the most specific scope mapping to each node in the syntax tree', async () => {
const grammar = new TreeSitterGrammar(atom.grammars, jsGrammarPath, {
parser: 'tree-sitter-javascript',
scopes: {
Expand All @@ -31,8 +31,11 @@ describe('TreeSitterLanguageMode', () => {
}
})

buffer.setLanguageMode(new TreeSitterLanguageMode({buffer, grammar}))
const languageMode = new TreeSitterLanguageMode({buffer, grammar})
buffer.setLanguageMode(languageMode)
buffer.setText('aa.bbb = cc(d.eee());')
await languageMode.reparsePromise

expectTokensToEqual(editor, [[
{text: 'aa.', scopes: ['source']},
{text: 'bbb', scopes: ['source', 'property']},
Expand All @@ -44,7 +47,7 @@ describe('TreeSitterLanguageMode', () => {
]])
})

it('can start or end multiple scopes at the same position', () => {
it('can start or end multiple scopes at the same position', async () => {
const grammar = new TreeSitterGrammar(atom.grammars, jsGrammarPath, {
parser: 'tree-sitter-javascript',
scopes: {
Expand All @@ -57,8 +60,11 @@ describe('TreeSitterLanguageMode', () => {
}
})

buffer.setLanguageMode(new TreeSitterLanguageMode({buffer, grammar}))
const languageMode = new TreeSitterLanguageMode({buffer, grammar})
buffer.setLanguageMode(languageMode)
buffer.setText('a = bb.ccc();')
await languageMode.reparsePromise

expectTokensToEqual(editor, [[
{text: 'a', scopes: ['source', 'variable']},
{text: ' = ', scopes: ['source']},
Expand All @@ -70,7 +76,7 @@ describe('TreeSitterLanguageMode', () => {
]])
})

it('can resume highlighting on a line that starts with whitespace', () => {
it('can resume highlighting on a line that starts with whitespace', async () => {
const grammar = new TreeSitterGrammar(atom.grammars, jsGrammarPath, {
parser: 'tree-sitter-javascript',
scopes: {
Expand All @@ -80,8 +86,11 @@ describe('TreeSitterLanguageMode', () => {
}
})

buffer.setLanguageMode(new TreeSitterLanguageMode({buffer, grammar}))
const languageMode = new TreeSitterLanguageMode({buffer, grammar})
buffer.setLanguageMode(languageMode)
buffer.setText('a\n .b();')
await languageMode.reparsePromise

expectTokensToEqual(editor, [
[
{text: 'a', scopes: ['variable']},
Expand All @@ -95,7 +104,7 @@ describe('TreeSitterLanguageMode', () => {
])
})

it('correctly skips over tokens with zero size', () => {
it('correctly skips over tokens with zero size', async () => {
const grammar = new TreeSitterGrammar(atom.grammars, jsGrammarPath, {
parser: 'tree-sitter-c',
scopes: {
Expand All @@ -107,10 +116,11 @@ describe('TreeSitterLanguageMode', () => {
const languageMode = new TreeSitterLanguageMode({buffer, grammar})
buffer.setLanguageMode(languageMode)
buffer.setText('int main() {\n int a\n int b;\n}');
await languageMode.reparsePromise

editor.screenLineForScreenRow(0)
expect(
languageMode.document.rootNode.descendantForPosition(Point(1, 2), Point(1, 6)).toString()
languageMode.tree.rootNode.descendantForPosition(Point(1, 2), Point(1, 6)).toString()
).toBe('(declaration (primitive_type) (identifier) (MISSING))')

expectTokensToEqual(editor, [
Expand Down Expand Up @@ -139,7 +149,7 @@ describe('TreeSitterLanguageMode', () => {
])
})

it('updates lines\' highlighting when they are affected by distant changes', () => {
it('updates lines\' highlighting when they are affected by distant changes', async () => {
const grammar = new TreeSitterGrammar(atom.grammars, jsGrammarPath, {
parser: 'tree-sitter-javascript',
scopes: {
Expand All @@ -148,10 +158,12 @@ describe('TreeSitterLanguageMode', () => {
}
})

buffer.setLanguageMode(new TreeSitterLanguageMode({buffer, grammar}))
const languageMode = new TreeSitterLanguageMode({buffer, grammar})
buffer.setLanguageMode(languageMode)
buffer.setText('a(\nb,\nc\n')
await languageMode.reparsePromise

// missing closing paren
buffer.setText('a(\nb,\nc\n')
expectTokensToEqual(editor, [
[{text: 'a(', scopes: []}],
[{text: 'b,', scopes: []}],
Expand All @@ -160,6 +172,7 @@ describe('TreeSitterLanguageMode', () => {
])

buffer.append(')')
await languageMode.reparsePromise
expectTokensToEqual(editor, [
[
{text: 'a', scopes: ['function']},
Expand All @@ -171,7 +184,7 @@ describe('TreeSitterLanguageMode', () => {
])
})

it('handles edits after tokens that end between CR and LF characters (regression)', () => {
it('handles edits after tokens that end between CR and LF characters (regression)', async () => {
const grammar = new TreeSitterGrammar(atom.grammars, jsGrammarPath, {
parser: 'tree-sitter-javascript',
scopes: {
Expand All @@ -181,13 +194,14 @@ describe('TreeSitterLanguageMode', () => {
}
})

buffer.setLanguageMode(new TreeSitterLanguageMode({buffer, grammar}))

const languageMode = new TreeSitterLanguageMode({buffer, grammar})
buffer.setLanguageMode(languageMode)
buffer.setText([
'// abc',
'',
'a("b").c'
].join('\r\n'))
await languageMode.reparsePromise

expectTokensToEqual(editor, [
[{text: '// abc', scopes: ['comment']}],
Expand All @@ -201,6 +215,7 @@ describe('TreeSitterLanguageMode', () => {
])

buffer.insert([2, 0], ' ')
await languageMode.reparsePromise
expectTokensToEqual(editor, [
[{text: '// abc', scopes: ['comment']}],
[{text: '', scopes: []}],
Expand All @@ -220,7 +235,7 @@ describe('TreeSitterLanguageMode', () => {
editor.displayLayer.reset({foldCharacter: '…'})
})

it('can fold nodes that start and end with specified tokens', () => {
it('can fold nodes that start and end with specified tokens', async () => {
const grammar = new TreeSitterGrammar(atom.grammars, jsGrammarPath, {
parser: 'tree-sitter-javascript',
folds: [
Expand All @@ -235,7 +250,8 @@ describe('TreeSitterLanguageMode', () => {
]
})

buffer.setLanguageMode(new TreeSitterLanguageMode({buffer, grammar}))
const languageMode = new TreeSitterLanguageMode({buffer, grammar})
buffer.setLanguageMode(languageMode)
buffer.setText(dedent `
module.exports =
class A {
Expand All @@ -246,6 +262,7 @@ describe('TreeSitterLanguageMode', () => {
}
}
`)
await languageMode.reparsePromise

editor.screenLineForScreenRow(0)

Expand Down Expand Up @@ -275,7 +292,7 @@ describe('TreeSitterLanguageMode', () => {
`)
})

it('can fold nodes of specified types', () => {
it('can fold nodes of specified types', async () => {
const grammar = new TreeSitterGrammar(atom.grammars, jsGrammarPath, {
parser: 'tree-sitter-javascript',
folds: [
Expand All @@ -296,7 +313,8 @@ describe('TreeSitterLanguageMode', () => {
]
})

buffer.setLanguageMode(new TreeSitterLanguageMode({buffer, grammar}))
const languageMode = new TreeSitterLanguageMode({buffer, grammar})
buffer.setLanguageMode(languageMode)
buffer.setText(dedent `
const element1 = <Element
className='submit'
Expand All @@ -307,6 +325,7 @@ describe('TreeSitterLanguageMode', () => {
<span>world</span>
</Element>
`)
await languageMode.reparsePromise

editor.screenLineForScreenRow(0)

Expand Down Expand Up @@ -336,7 +355,7 @@ describe('TreeSitterLanguageMode', () => {
`)
})

it('can fold entire nodes when no start or end parameters are specified', () => {
it('can fold entire nodes when no start or end parameters are specified', async () => {
const grammar = new TreeSitterGrammar(atom.grammars, jsGrammarPath, {
parser: 'tree-sitter-javascript',
folds: [
Expand All @@ -346,7 +365,8 @@ describe('TreeSitterLanguageMode', () => {
]
})

buffer.setLanguageMode(new TreeSitterLanguageMode({buffer, grammar}))
const languageMode = new TreeSitterLanguageMode({buffer, grammar})
buffer.setLanguageMode(languageMode)
buffer.setText(dedent `
/**
* Important
Expand All @@ -355,6 +375,7 @@ describe('TreeSitterLanguageMode', () => {
Also important
*/
`)
await languageMode.reparsePromise

editor.screenLineForScreenRow(0)

Expand All @@ -379,7 +400,7 @@ describe('TreeSitterLanguageMode', () => {
`)
})

it('tries each folding strategy for a given node in the order specified', () => {
it('tries each folding strategy for a given node in the order specified', async () => {
const grammar = new TreeSitterGrammar(atom.grammars, cGrammarPath, {
parser: 'tree-sitter-c',
folds: [
Expand All @@ -405,8 +426,8 @@ describe('TreeSitterLanguageMode', () => {
]
})

buffer.setLanguageMode(new TreeSitterLanguageMode({buffer, grammar}))

const languageMode = new TreeSitterLanguageMode({buffer, grammar})
buffer.setLanguageMode(languageMode)
buffer.setText(dedent `
#ifndef FOO_H_
#define FOO_H_
Expand All @@ -430,6 +451,7 @@ describe('TreeSitterLanguageMode', () => {

#endif
`)
await languageMode.reparsePromise

editor.screenLineForScreenRow(0)

Expand Down Expand Up @@ -504,8 +526,6 @@ describe('TreeSitterLanguageMode', () => {
]
})

buffer.setLanguageMode(new TreeSitterLanguageMode({buffer, grammar}))

buffer.setText(dedent `
def ab():
print 'a'
Expand All @@ -515,6 +535,7 @@ describe('TreeSitterLanguageMode', () => {
print 'c'
print 'd'
`)
buffer.setLanguageMode(new TreeSitterLanguageMode({buffer, grammar}))

editor.screenLineForScreenRow(0)

Expand All @@ -537,9 +558,8 @@ describe('TreeSitterLanguageMode', () => {
parser: 'tree-sitter-javascript'
})

buffer.setLanguageMode(new TreeSitterLanguageMode({buffer, grammar}))

buffer.setText('foo({bar: baz});')
buffer.setLanguageMode(new TreeSitterLanguageMode({buffer, grammar}))

editor.screenLineForScreenRow(0)
expect(editor.scopeDescriptorForBufferPosition([0, 6]).getScopesArray()).toEqual([
Expand All @@ -562,13 +582,13 @@ describe('TreeSitterLanguageMode', () => {
scopes: {'program': 'source'}
})

buffer.setLanguageMode(new TreeSitterLanguageMode({buffer, grammar}))
buffer.setText(dedent `
function a (b, c, d) {
eee.f()
g()
}
`)
buffer.setLanguageMode(new TreeSitterLanguageMode({buffer, grammar}))

editor.screenLineForScreenRow(0)

Expand Down
Loading