diff --git a/packages/language-rust-bundled/lib/main.js b/packages/language-rust-bundled/lib/main.js index 982a9b3389d..7a996b68940 100644 --- a/packages/language-rust-bundled/lib/main.js +++ b/packages/language-rust-bundled/lib/main.js @@ -2,8 +2,13 @@ exports.activate = function() { for (const nodeType of ['macro_invocation', 'macro_rule']) { atom.grammars.addInjectionPoint('source.rust', { type: nodeType, - language() { return 'rust'; }, - content(node) { return node.lastChild; }, + language() { + return 'rust'; + }, + content(node) { + return node.lastChild; + }, + includeChildren: true }); } }; diff --git a/spec/tree-sitter-language-mode-spec.js b/spec/tree-sitter-language-mode-spec.js index 117132bf2d7..8919a54eccd 100644 --- a/spec/tree-sitter-language-mode-spec.js +++ b/spec/tree-sitter-language-mode-spec.js @@ -27,6 +27,9 @@ const ejsGrammarPath = require.resolve( const rubyGrammarPath = require.resolve( 'language-ruby/grammars/tree-sitter-ruby.cson' ); +const rustGrammarPath = require.resolve( + 'language-rust-bundled/grammars/tree-sitter-rust.cson' +); describe('TreeSitterLanguageMode', () => { let editor, buffer; @@ -831,6 +834,81 @@ describe('TreeSitterLanguageMode', () => { ]); }); + it('respects the `includeChildren` property of injection points', async () => { + const rustGrammar = new TreeSitterGrammar( + atom.grammars, + rustGrammarPath, + { + scopeName: 'rust', + parser: 'tree-sitter-rust', + scopes: { + identifier: 'variable', + field_identifier: 'property', + 'call_expression > field_expression > field_identifier': + 'function', + 'macro_invocation > identifier': 'macro' + }, + injectionRegExp: 'rust', + injectionPoints: [ + { + type: 'macro_invocation', + language() { + return 'rust'; + }, + content(node) { + return node.lastChild; + }, + + // The tokens within a `token_tree` are all parsed as separate + // children of the `token_tree`. By default, when adding a language + // injection for a node, the node's children's ranges would be + // excluded from the injection. But for this injection point + // (parsing token trees as rust code), we want to reparse all of the + // content of the token tree. + includeChildren: true + } + ] + } + ); + + atom.grammars.addGrammar(rustGrammar); + + // Macro call within another macro call. + buffer.setText('assert_eq!(a.b.c(), vec![d.e()]); f.g();'); + + const languageMode = new TreeSitterLanguageMode({ + buffer, + grammar: rustGrammar, + grammars: atom.grammars + }); + buffer.setLanguageMode(languageMode); + + // There should not be duplicate scopes due to the root layer + // and for the injected rust layer. + expectTokensToEqual(editor, [ + [ + { text: 'assert_eq', scopes: ['macro'] }, + { text: '!(', scopes: [] }, + { text: 'a', scopes: ['variable'] }, + { text: '.', scopes: [] }, + { text: 'b', scopes: ['property'] }, + { text: '.', scopes: [] }, + { text: 'c', scopes: ['function'] }, + { text: '(), ', scopes: [] }, + { text: 'vec', scopes: ['macro'] }, + { text: '![', scopes: [] }, + { text: 'd', scopes: ['variable'] }, + { text: '.', scopes: [] }, + { text: 'e', scopes: ['function'] }, + { text: '()]); ', scopes: [] }, + { text: 'f', scopes: ['variable'] }, + { text: '.', scopes: [] }, + { text: 'g', scopes: ['function'] }, + { text: '();', scopes: [] } + ] + ]); + }); + it('notifies onDidTokenize listeners the first time all syntax highlighting is done', async () => { const promise = new Promise(resolve => { editor.onDidTokenize(event => { diff --git a/src/tree-sitter-language-mode.js b/src/tree-sitter-language-mode.js index 52b93b508b8..7d6ecf8cbd9 100644 --- a/src/tree-sitter-language-mode.js +++ b/src/tree-sitter-language-mode.js @@ -32,7 +32,7 @@ class TreeSitterLanguageMode { this.config = config; this.grammarRegistry = grammars; this.parser = new Parser(); - this.rootLanguageLayer = new LanguageLayer(this, grammar); + this.rootLanguageLayer = new LanguageLayer(this, grammar, 0, true); this.injectionsMarkerLayer = buffer.addMarkerLayer(); if (syncTimeoutMicros != null) { @@ -637,13 +637,14 @@ class TreeSitterLanguageMode { } class LanguageLayer { - constructor(languageMode, grammar, contentChildTypes) { + constructor(languageMode, grammar, depth, isOpaque) { this.languageMode = languageMode; this.grammar = grammar; this.tree = null; this.currentParsePromise = null; this.patchSinceCurrentParseStarted = null; - this.contentChildTypes = contentChildTypes; + this.depth = depth; + this.isOpaque = isOpaque; } buildHighlightIterator() { @@ -885,7 +886,8 @@ class LanguageLayer { marker.languageLayer = new LanguageLayer( this.languageMode, grammar, - injectionPoint.contentChildTypes + this.depth + 1, + injectionPoint.includeChildren ); marker.parentLanguageLayer = this; } @@ -895,7 +897,8 @@ class LanguageLayer { new NodeRangeSet( nodeRangeSet, injectionNodes, - injectionPoint.newlinesBetween + injectionPoint.newlinesBetween, + injectionPoint.includeChildren ) ); } @@ -910,7 +913,6 @@ class LanguageLayer { } if (markersToUpdate.size > 0) { - this.lastUpdateWasAsync = true; const promises = []; for (const [marker, nodeRangeSet] of markersToUpdate) { promises.push(marker.languageLayer.update(nodeRangeSet)); @@ -938,6 +940,7 @@ class HighlightIterator { constructor(languageMode) { this.languageMode = languageMode; this.iterators = null; + this.opaqueLayerDepth = 0; } seek(targetPosition, endRow) { @@ -947,55 +950,97 @@ class HighlightIterator { } ); - this.iterators = [ - this.languageMode.rootLanguageLayer.buildHighlightIterator() - ]; - for (const marker of injectionMarkers) { - this.iterators.push(marker.languageLayer.buildHighlightIterator()); - } - this.iterators.sort((a, b) => b.getIndex() - a.getIndex()); - const containingTags = []; const containingTagStartIndices = []; const targetIndex = this.languageMode.buffer.characterIndexForPosition( targetPosition ); - for (let i = this.iterators.length - 1; i >= 0; i--) { - this.iterators[i].seek( - targetIndex, - containingTags, - containingTagStartIndices - ); + + this.iterators = []; + const iterator = this.languageMode.rootLanguageLayer.buildHighlightIterator(); + if (iterator.seek(targetIndex, containingTags, containingTagStartIndices)) { + this.iterators.push(iterator); } - this.iterators.sort((a, b) => b.getIndex() - a.getIndex()); + for (const marker of injectionMarkers) { + const iterator = marker.languageLayer.buildHighlightIterator(); + if ( + iterator.seek(targetIndex, containingTags, containingTagStartIndices) + ) { + this.iterators.push(iterator); + } + } + this.iterators.sort((a, b) => b.compare(a)); return containingTags; } moveToSuccessor() { - const lastIndex = this.iterators.length - 1; - const leader = this.iterators[lastIndex]; - leader.moveToSuccessor(); - const leaderCharIndex = leader.getIndex(); - let i = lastIndex; - while (i > 0 && this.iterators[i - 1].getIndex() < leaderCharIndex) i--; - if (i < lastIndex) this.iterators.splice(i, 0, this.iterators.pop()); + let leader = last(this.iterators); + if (leader.moveToSuccessor()) { + const leaderIndex = this.iterators.length - 1; + let i = leaderIndex; + while (i > 0 && this.iterators[i - 1].compare(leader) < 0) i--; + if (i < leaderIndex) { + this.iterators.splice(i, 0, this.iterators.pop()); + this.trackLayerDepth(); + } + } else { + this.iterators.pop(); + this.trackLayerDepth(); + } + } + + trackLayerDepth() { + let i = this.iterators.length - 1; + let iterator = this.iterators[i]; + if (!iterator) return; + const offset = iterator.getOffset(); + if (iterator.languageLayer.isOpaque) { + this.opaqueLayerDepth = iterator.languageLayer.depth; + } + + while (i > 0) { + i--; + iterator = this.iterators[i]; + if (iterator.startOffset > offset) break; + if (iterator.languageLayer.isOpaque) { + const { depth } = iterator.languageLayer; + if (depth > this.opaqueLayerDepth) { + this.opaqueLayerDepth = depth; + } + } + } } getPosition() { - return last(this.iterators).getPosition(); + const iterator = last(this.iterators); + if (iterator) { + return iterator.getPosition(); + } else { + return Point.INFINITY; + } } getCloseScopeIds() { - return last(this.iterators).getCloseScopeIds(); + const iterator = last(this.iterators); + if (iterator && iterator.languageLayer.depth >= this.opaqueLayerDepth) { + return iterator.getCloseScopeIds(); + } else { + return []; + } } getOpenScopeIds() { - return last(this.iterators).getOpenScopeIds(); + const iterator = last(this.iterators); + if (iterator && iterator.languageLayer.depth >= this.opaqueLayerDepth) { + return iterator.getOpenScopeIds(); + } else { + return []; + } } logState() { const iterator = last(this.iterators); - if (iterator.treeCursor) { + if (iterator && iterator.treeCursor) { console.log( iterator.getPosition(), iterator.treeCursor.nodeType, @@ -1029,6 +1074,8 @@ class LayerHighlightIterator { this.atEnd = false; this.treeCursor = treeCursor; + this.startOffset = this.treeCursor.startIndex; + // In order to determine which selectors match its current node, the iterator maintains // a list of the current node's ancestors. Because the selectors can use the `:nth-child` // pseudo-class, each node's child index is also stored. @@ -1046,7 +1093,6 @@ class LayerHighlightIterator { seek(targetIndex, containingTags, containingTagStartIndices) { while (this.treeCursor.gotoParent()) {} - this.done = false; this.atEnd = true; this.closeTags.length = 0; this.openTags.length = 0; @@ -1057,8 +1103,7 @@ class LayerHighlightIterator { const containingTagEndIndices = []; if (targetIndex >= this.treeCursor.endIndex) { - this.done = true; - return; + return false; } let childIndex = -1; @@ -1099,14 +1144,14 @@ class LayerHighlightIterator { } } - return containingTags; + return true; } moveToSuccessor() { this.closeTags.length = 0; this.openTags.length = 0; - while (!this.done && !this.closeTags.length && !this.openTags.length) { + while (!this.closeTags.length && !this.openTags.length) { if (this.atEnd) { if (this._moveRight()) { const scopeId = this._currentScopeId(); @@ -1116,7 +1161,7 @@ class LayerHighlightIterator { } else if (this._moveUp(true)) { this.atEnd = true; } else { - this.done = true; + return false; } } else if (!this._moveDown()) { const scopeId = this._currentScopeId(); @@ -1125,28 +1170,34 @@ class LayerHighlightIterator { this._moveUp(false); } } + + return true; } getPosition() { - if (this.done) { - return Point.INFINITY; - } else if (this.atEnd) { + if (this.atEnd) { return this.treeCursor.endPosition; } else { return this.treeCursor.startPosition; } } - getIndex() { - if (this.done) { - return Infinity; - } else if (this.atEnd) { + getOffset() { + if (this.atEnd) { return this.treeCursor.endIndex; } else { return this.treeCursor.startIndex; } } + compare(other) { + let result = this.getOffset() - other.getOffset(); + if (result !== 0) return result; + if (this.atEnd && !other.atEnd) return -1; + if (other.atEnd && !this.atEnd) return 1; + return this.depth - other.depth; + } + getCloseScopeIds() { return this.closeTags.slice(); } @@ -1156,6 +1207,7 @@ class LayerHighlightIterator { } // Private methods + _moveUp(atLastChild) { let result = false; const { endIndex } = this.treeCursor; @@ -1264,7 +1316,7 @@ class NullHighlightIterator { return []; } moveToSuccessor() {} - getIndex() { + getOffset() { return Infinity; } getPosition() { @@ -1279,10 +1331,11 @@ class NullHighlightIterator { } class NodeRangeSet { - constructor(previous, nodes, newlinesBetween) { + constructor(previous, nodes, newlinesBetween, includeChildren) { this.previous = previous; this.nodes = nodes; this.newlinesBetween = newlinesBetween; + this.includeChildren = includeChildren; } getRanges(buffer) { @@ -1293,18 +1346,20 @@ class NodeRangeSet { let position = node.startPosition; let index = node.startIndex; - for (const child of node.children) { - const nextIndex = child.startIndex; - if (nextIndex > index) { - this._pushRange(buffer, previousRanges, result, { - startIndex: index, - endIndex: nextIndex, - startPosition: position, - endPosition: child.startPosition - }); + if (!this.includeChildren) { + for (const child of node.children) { + const nextIndex = child.startIndex; + if (nextIndex > index) { + this._pushRange(buffer, previousRanges, result, { + startIndex: index, + endIndex: nextIndex, + startPosition: position, + endPosition: child.startPosition + }); + } + position = child.endPosition; + index = child.endIndex; } - position = child.endPosition; - index = child.endIndex; } if (node.endIndex > index) {