Skip to content

Commit

Permalink
Reduce IModelLinesTokens shape
Browse files Browse the repository at this point in the history
  • Loading branch information
@alexdima
alexdima committed Jul 3, 2019
1 parent 8a5f705 commit d1c16cf
Show file tree
Hide file tree
Showing 3 changed files with 80 additions and 108 deletions.
60 changes: 5 additions & 55 deletions src/vs/editor/common/model/textModel.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,9 +23,9 @@ import { IntervalNode, IntervalTree, getNodeIsInOverviewRuler, recomputeMaxEnd }
import { PieceTreeTextBufferBuilder } from 'vs/editor/common/model/pieceTreeTextBuffer/pieceTreeTextBufferBuilder';
import { IModelContentChangedEvent, IModelDecorationsChangedEvent, IModelLanguageChangedEvent, IModelLanguageConfigurationChangedEvent, IModelOptionsChangedEvent, IModelTokensChangedEvent, InternalModelContentChangeEvent, ModelRawChange, ModelRawContentChangedEvent, ModelRawEOLChanged, ModelRawFlush, ModelRawLineChanged, ModelRawLinesDeleted, ModelRawLinesInserted } from 'vs/editor/common/model/textModelEvents';
import { SearchData, SearchParams, TextModelSearch } from 'vs/editor/common/model/textModelSearch';
import { ModelLinesTokens, ModelTokensChangedEventBuilder, IModelLinesTokens, safeTokenize } from 'vs/editor/common/model/textModelTokens';
import { ModelLinesTokens, ModelTokensChangedEventBuilder, IModelLinesTokens } from 'vs/editor/common/model/textModelTokens';
import { getWordAtText } from 'vs/editor/common/model/wordHelper';
import { IState, LanguageId, LanguageIdentifier, TokenizationRegistry, FormattingOptions } from 'vs/editor/common/modes';
import { LanguageId, LanguageIdentifier, TokenizationRegistry, FormattingOptions } from 'vs/editor/common/modes';
import { LanguageConfigurationRegistry } from 'vs/editor/common/modes/languageConfigurationRegistry';
import { NULL_LANGUAGE_IDENTIFIER } from 'vs/editor/common/modes/nullMode';
import { ignoreBracketsInToken } from 'vs/editor/common/modes/supports';
Expand Down Expand Up @@ -1342,7 +1342,7 @@ export class TextModel extends Disposable implements model.ITextModel {
const change = contentChanges[i];
const [eolCount, firstLineLength] = TextModel._eolCount(change.text);
try {
this._tokens.applyEdits(change.range, eolCount, firstLineLength);
this._tokens.store.applyEdits(change.range, eolCount, firstLineLength);
} catch (err) {
// emergency recovery => reset tokens
this._tokens = new ModelLinesTokens(this._languageIdentifier, this._tokens.tokenizationSupport);
Expand Down Expand Up @@ -1776,61 +1776,11 @@ export class TextModel extends Disposable implements model.ITextModel {
//#region Tokenization

public tokenizeViewport(startLineNumber: number, endLineNumber: number): void {
if (!this._tokens.tokenizationSupport) {
// nothing to do
return;
}

startLineNumber = Math.max(1, startLineNumber);
endLineNumber = Math.min(this.getLineCount(), endLineNumber);

if (endLineNumber <= this._tokens.invalidLineStartIndex) {
// nothing to do
return;
}

if (startLineNumber <= this._tokens.invalidLineStartIndex) {
// tokenization has reached the viewport start...
this.forceTokenization(endLineNumber);
return;
}

let nonWhitespaceColumn = this.getLineFirstNonWhitespaceColumn(startLineNumber);
let fakeLines: string[] = [];
let initialState: IState | null = null;
for (let i = startLineNumber - 1; nonWhitespaceColumn > 0 && i >= 1; i--) {
let newNonWhitespaceIndex = this.getLineFirstNonWhitespaceColumn(i);

if (newNonWhitespaceIndex === 0) {
continue;
}

if (newNonWhitespaceIndex < nonWhitespaceColumn) {
initialState = this._tokens.getState(i - 1);
if (initialState) {
break;
}
fakeLines.push(this.getLineContent(i));
nonWhitespaceColumn = newNonWhitespaceIndex;
}
}

if (!initialState) {
initialState = this._tokens.tokenizationSupport.getInitialState();
}

let state = initialState;
for (let i = fakeLines.length - 1; i >= 0; i--) {
let r = safeTokenize(this._languageIdentifier, this._tokens.tokenizationSupport, fakeLines[i], state);
if (r) {
state = r.endState;
} else {
state = initialState;
}
}

const eventBuilder = new ModelTokensChangedEventBuilder();
this._tokens.fakeTokenizeLines(this._buffer, eventBuilder, state, startLineNumber, endLineNumber);
this._tokens.tokenizeViewport(this._buffer, eventBuilder, startLineNumber, endLineNumber);

const e = eventBuilder.build();
if (e) {
Expand Down Expand Up @@ -1896,7 +1846,7 @@ export class TextModel extends Disposable implements model.ITextModel {

private _getLineTokens(lineNumber: number): LineTokens {
const lineText = this._buffer.getLineContent(lineNumber);
return this._tokens.getTokens(this._languageIdentifier.id, lineNumber - 1, lineText);
return this._tokens.store.getTokens(this._languageIdentifier.id, lineNumber - 1, lineText);
}

public getLanguageIdentifier(): LanguageIdentifier {
Expand Down
122 changes: 72 additions & 50 deletions src/vs/editor/common/model/textModelTokens.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -390,32 +390,32 @@ class TokensStore {

export interface IModelLinesTokens {
readonly tokenizationSupport: ITokenizationSupport | null;
readonly store: TokensStore;
readonly invalidLineStartIndex: number;

setTokens(topLevelLanguageId: LanguageId, lineIndex: number, lineTextLength: number, tokens: Uint32Array): void;
getTokens(topLevelLanguageId: LanguageId, lineIndex: number, lineText: string): LineTokens;

isCheapToTokenize(lineNumber: number): boolean;
hasLinesToTokenize(buffer: ITextBuffer): boolean;
getState(lineIndex: number): IState | null;
applyEdits(range: Range, eolCount: number, firstLineLength: number): void;

tokenizeOneInvalidLine(buffer: ITextBuffer, eventBuilder: ModelTokensChangedEventBuilder): number;
updateTokensUntilLine(buffer: ITextBuffer, eventBuilder: ModelTokensChangedEventBuilder, lineNumber: number): void;
fakeTokenizeLines(buffer: ITextBuffer, eventBuilder: ModelTokensChangedEventBuilder, initialState: IState, startLineNumber: number, endLineNumber: number): void;
tokenizeViewport(buffer: ITextBuffer, eventBuilder: ModelTokensChangedEventBuilder, startLineNumber: number, endLineNumber: number): void;

_getAllStates(linesLength: number): (IState | null)[];
_getAllInvalid(linesLength: number): number[];
}

export class ModelLinesTokens implements IModelLinesTokens {

public readonly languageIdentifier: LanguageIdentifier;
private readonly _languageIdentifier: LanguageIdentifier;
public readonly tokenizationSupport: ITokenizationSupport | null;
public readonly store: TokensStore;

public get invalidLineStartIndex() {
return this.store.invalidLineStartIndex;
}

constructor(languageIdentifier: LanguageIdentifier, tokenizationSupport: ITokenizationSupport | null) {
this.languageIdentifier = languageIdentifier;
this._languageIdentifier = languageIdentifier;
this.tokenizationSupport = tokenizationSupport;

let initialState: IState | null = null;
Expand All @@ -430,14 +430,6 @@ export class ModelLinesTokens implements IModelLinesTokens {
this.store = new TokensStore(initialState);
}

public get invalidLineStartIndex() {
return this.store.invalidLineStartIndex;
}

public getTokens(topLevelLanguageId: LanguageId, lineIndex: number, lineText: string): LineTokens {
return this.store.getTokens(topLevelLanguageId, lineIndex, lineText);
}

public isCheapToTokenize(lineNumber: number): boolean {
const firstInvalidLineNumber = this.store.invalidLineStartIndex + 1;
return (firstInvalidLineNumber >= lineNumber);
Expand All @@ -447,30 +439,6 @@ export class ModelLinesTokens implements IModelLinesTokens {
return (this.store.invalidLineStartIndex < buffer.getLineCount());
}

public getState(lineIndex: number): IState | null {
return this.store.getState(lineIndex);
}

public setTokens(topLevelLanguageId: LanguageId, lineIndex: number, lineTextLength: number, tokens: Uint32Array): void {
this.store.setTokens(topLevelLanguageId, lineIndex, lineTextLength, tokens);
}

private _setState(lineIndex: number, state: IState): void {
this.store.setState(lineIndex, state);
}

//#region Editing

public applyEdits(range: Range, eolCount: number, firstLineLength: number): void {
this.store.applyEdits(range, eolCount, firstLineLength);
}

private _invalidateLine(lineIndex: number): void {
this.store.invalidateLine(lineIndex);
}

//#endregion

//#region Tokenization

public tokenizeOneInvalidLine(buffer: ITextBuffer, eventBuilder: ModelTokensChangedEventBuilder): number {
Expand All @@ -494,31 +462,85 @@ export class ModelLinesTokens implements IModelLinesTokens {
// Validate all states up to and including endLineIndex
for (let lineIndex = this.store.invalidLineStartIndex; lineIndex <= endLineIndex; lineIndex++) {
const text = buffer.getLineContent(lineIndex + 1);
const lineStartState = this.getState(lineIndex);
const lineStartState = this.store.getState(lineIndex);

const r = safeTokenize(this.languageIdentifier, this.tokenizationSupport, text, lineStartState!);
this.store.setGoodTokens(this.languageIdentifier.id, linesLength, lineIndex, text, r);
const r = safeTokenize(this._languageIdentifier, this.tokenizationSupport, text, lineStartState!);
this.store.setGoodTokens(this._languageIdentifier.id, linesLength, lineIndex, text, r);
eventBuilder.registerChangedTokens(lineIndex + 1);
lineIndex = this.store.invalidLineStartIndex - 1; // -1 because the outer loop increments it
}
}

public fakeTokenizeLines(buffer: ITextBuffer, eventBuilder: ModelTokensChangedEventBuilder, initialState: IState, startLineNumber: number, endLineNumber: number): void {
public tokenizeViewport(buffer: ITextBuffer, eventBuilder: ModelTokensChangedEventBuilder, startLineNumber: number, endLineNumber: number): void {
if (!this.tokenizationSupport) {
// nothing to do
return;
}

if (endLineNumber <= this.invalidLineStartIndex) {
// nothing to do
return;
}

if (startLineNumber <= this.invalidLineStartIndex) {
// tokenization has reached the viewport start...
this.updateTokensUntilLine(buffer, eventBuilder, endLineNumber);
return;
}

let nonWhitespaceColumn = buffer.getLineFirstNonWhitespaceColumn(startLineNumber);
let fakeLines: string[] = [];
let initialState: IState | null = null;
for (let i = startLineNumber - 1; nonWhitespaceColumn > 0 && i >= 1; i--) {
let newNonWhitespaceIndex = buffer.getLineFirstNonWhitespaceColumn(i);

if (newNonWhitespaceIndex === 0) {
continue;
}

if (newNonWhitespaceIndex < nonWhitespaceColumn) {
initialState = this.store.getState(i - 1);
if (initialState) {
break;
}
fakeLines.push(buffer.getLineContent(i));
nonWhitespaceColumn = newNonWhitespaceIndex;
}
}

if (!initialState) {
initialState = this.tokenizationSupport.getInitialState();
}

let state = initialState;
for (let i = fakeLines.length - 1; i >= 0; i--) {
let r = safeTokenize(this._languageIdentifier, this.tokenizationSupport, fakeLines[i], state);
if (r) {
state = r.endState;
} else {
state = initialState;
}
}

this._fakeTokenizeLines(buffer, eventBuilder, state, startLineNumber, endLineNumber);
}

private _fakeTokenizeLines(buffer: ITextBuffer, eventBuilder: ModelTokensChangedEventBuilder, initialState: IState, startLineNumber: number, endLineNumber: number): void {
if (!this.tokenizationSupport) {
return;
}

let state = initialState;
for (let i = startLineNumber; i <= endLineNumber; i++) {
let text = buffer.getLineContent(i);
let r = safeTokenize(this.languageIdentifier, this.tokenizationSupport, text, state);
let r = safeTokenize(this._languageIdentifier, this.tokenizationSupport, text, state);
if (r) {
this.setTokens(this.languageIdentifier.id, i - 1, text.length, r.tokens);
this.store.setTokens(this._languageIdentifier.id, i - 1, text.length, r.tokens);

// We cannot trust these states/tokens to be valid!
// (see https://github.com/Microsoft/vscode/issues/67607)
this._invalidateLine(i - 1);
this._setState(i - 1, state);
this.store.invalidateLine(i - 1);
this.store.setState(i - 1, state);
state = r.endState;
eventBuilder.registerChangedTokens(i);
} else {
Expand All @@ -532,7 +554,7 @@ export class ModelLinesTokens implements IModelLinesTokens {
_getAllStates(linesLength: number): (IState | null)[] {
const r: (IState | null)[] = [];
for (let i = 0; i < linesLength; i++) {
r[i] = this.getState(i);
r[i] = this.store.getState(i);
}
r[linesLength] = this.store._lastState;
return r;
Expand All @@ -549,7 +571,7 @@ export class ModelLinesTokens implements IModelLinesTokens {
}
}

export function safeTokenize(languageIdentifier: LanguageIdentifier, tokenizationSupport: ITokenizationSupport | null, text: string, state: IState): TokenizationResult2 {
function safeTokenize(languageIdentifier: LanguageIdentifier, tokenizationSupport: ITokenizationSupport | null, text: string, state: IState): TokenizationResult2 {
let r: TokenizationResult2 | null = null;

if (tokenizationSupport) {
Expand Down
6 changes: 3 additions & 3 deletions src/vs/editor/test/common/model/model.line.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -110,7 +110,7 @@ suite('ModelLinesTokens', () => {
for (let lineIndex = 0; lineIndex < initial.length; lineIndex++) {
const lineTokens = initial[lineIndex].tokens;
const lineTextLength = model.getLineMaxColumn(lineIndex + 1) - 1;
model._tokens.setTokens(0, lineIndex, lineTextLength, TestToken.toTokens(lineTokens));
model._tokens.store.setTokens(0, lineIndex, lineTextLength, TestToken.toTokens(lineTokens));
}

model.applyEdits(edits.map((ed) => ({
Expand Down Expand Up @@ -441,14 +441,14 @@ suite('ModelLinesTokens', () => {

test('insertion on empty line', () => {
const model = new TextModel('some text', TextModel.DEFAULT_CREATION_OPTIONS, new LanguageIdentifier('test', 0));
model._tokens.setTokens(0, 0, model.getLineMaxColumn(1) - 1, TestToken.toTokens([new TestToken(0, 1)]));
model._tokens.store.setTokens(0, 0, model.getLineMaxColumn(1) - 1, TestToken.toTokens([new TestToken(0, 1)]));

model.applyEdits([{
range: new Range(1, 1, 1, 10),
text: ''
}]);

model._tokens.setTokens(0, 0, model.getLineMaxColumn(1) - 1, new Uint32Array(0));
model._tokens.store.setTokens(0, 0, model.getLineMaxColumn(1) - 1, new Uint32Array(0));

model.applyEdits([{
range: new Range(1, 1, 1, 1),
Expand Down

0 comments on commit d1c16cf

Please sign in to comment.