From ef44e6778365479de6808e49662ffd8340908775 Mon Sep 17 00:00:00 2001 From: Alex Ross Date: Mon, 30 Sep 2024 15:18:32 +0200 Subject: [PATCH] Improve unusable slowness of tree sitter syntax highlighting (#229951) * Improve unusable slowness of tree sitter syntax highlighting Part of #229935 * Improve improvement as discussed with Martin * Fix score * Fix test error --- .../test-issue11_ts.json | 16 ++--- .../test-issue5431_ts.json | 12 ++-- .../test-issue5566_ts.json | 16 ++--- .../test-keywords_ts.json | 2 +- .../colorize-tree-sitter-results/test_ts.json | 10 +-- .../languages/highlights/typescript.scm | 10 +-- .../services/themes/common/colorThemeData.ts | 68 ++++++------------- .../test/node/tokenStyleResolving.test.ts | 3 + .../browser/treeSitterTokenizationFeature.ts | 4 +- 9 files changed, 55 insertions(+), 86 deletions(-) diff --git a/extensions/vscode-colorize-tests/test/colorize-tree-sitter-results/test-issue11_ts.json b/extensions/vscode-colorize-tests/test/colorize-tree-sitter-results/test-issue11_ts.json index 60fc1bedb81..6fb2651c7c5 100644 --- a/extensions/vscode-colorize-tests/test/colorize-tree-sitter-results/test-issue11_ts.json +++ b/extensions/vscode-colorize-tests/test/colorize-tree-sitter-results/test-issue11_ts.json @@ -1989,7 +1989,7 @@ }, { "c": "=>", - "t": "keyword.operator storage.type", + "t": "storage.type", "r": { "dark_plus": "storage.type: #569CD6", "light_plus": "storage.type: #0000FF", @@ -2507,7 +2507,7 @@ }, { "c": "=>", - "t": "keyword.operator storage.type", + "t": "storage.type", "r": { "dark_plus": "storage.type: #569CD6", "light_plus": "storage.type: #0000FF", @@ -2801,16 +2801,16 @@ }, { "c": "t", - "t": "variable variable.other.constant", + "t": "variable", "r": { - "dark_plus": "variable.other.constant: #4FC1FF", - "light_plus": "variable.other.constant: #0070C1", + "dark_plus": "variable: #9CDCFE", + "light_plus": "variable: #001080", "dark_vs": "default: #D4D4D4", "light_vs": "default: #000000", "hc_black": "variable: #9CDCFE", - "dark_modern": "variable.other.constant: #4FC1FF", - "hc_light": "variable.other.constant: #02715D", - "light_modern": "variable.other.constant: #0070C1" + "dark_modern": "variable: #9CDCFE", + "hc_light": "variable: #001080", + "light_modern": "variable: #001080" } }, { diff --git a/extensions/vscode-colorize-tests/test/colorize-tree-sitter-results/test-issue5431_ts.json b/extensions/vscode-colorize-tests/test/colorize-tree-sitter-results/test-issue5431_ts.json index d5508bf6b0a..2f99899f2a1 100644 --- a/extensions/vscode-colorize-tests/test/colorize-tree-sitter-results/test-issue5431_ts.json +++ b/extensions/vscode-colorize-tests/test/colorize-tree-sitter-results/test-issue5431_ts.json @@ -155,16 +155,16 @@ }, { "c": "timeRange", - "t": "variable variable.other.constant", + "t": "variable", "r": { - "dark_plus": "variable.other.constant: #4FC1FF", - "light_plus": "variable.other.constant: #0070C1", + "dark_plus": "variable: #9CDCFE", + "light_plus": "variable: #001080", "dark_vs": "default: #D4D4D4", "light_vs": "default: #000000", "hc_black": "variable: #9CDCFE", - "dark_modern": "variable.other.constant: #4FC1FF", - "hc_light": "variable.other.constant: #02715D", - "light_modern": "variable.other.constant: #0070C1" + "dark_modern": "variable: #9CDCFE", + "hc_light": "variable: #001080", + "light_modern": "variable: #001080" } }, { diff --git a/extensions/vscode-colorize-tests/test/colorize-tree-sitter-results/test-issue5566_ts.json b/extensions/vscode-colorize-tests/test/colorize-tree-sitter-results/test-issue5566_ts.json index edcd005f61c..e055f0ae7ca 100644 --- a/extensions/vscode-colorize-tests/test/colorize-tree-sitter-results/test-issue5566_ts.json +++ b/extensions/vscode-colorize-tests/test/colorize-tree-sitter-results/test-issue5566_ts.json @@ -85,16 +85,16 @@ }, { "c": "foo", - "t": "variable entity.name.function variable.other.constant", + "t": "variable entity.name.function", "r": { - "dark_plus": "variable.other.constant: #4FC1FF", - "light_plus": "variable.other.constant: #0070C1", + "dark_plus": "entity.name.function: #DCDCAA", + "light_plus": "entity.name.function: #795E26", "dark_vs": "default: #D4D4D4", "light_vs": "default: #000000", - "hc_black": "variable: #9CDCFE", - "dark_modern": "variable.other.constant: #4FC1FF", - "hc_light": "variable.other.constant: #02715D", - "light_modern": "variable.other.constant: #0070C1" + "hc_black": "entity.name.function: #DCDCAA", + "dark_modern": "entity.name.function: #DCDCAA", + "hc_light": "entity.name.function: #5E2CBC", + "light_modern": "entity.name.function: #795E26" } }, { @@ -169,7 +169,7 @@ }, { "c": "=>", - "t": "keyword.operator storage.type", + "t": "storage.type", "r": { "dark_plus": "storage.type: #569CD6", "light_plus": "storage.type: #0000FF", diff --git a/extensions/vscode-colorize-tests/test/colorize-tree-sitter-results/test-keywords_ts.json b/extensions/vscode-colorize-tests/test/colorize-tree-sitter-results/test-keywords_ts.json index 88e4148cc63..f7b50872f0a 100644 --- a/extensions/vscode-colorize-tests/test/colorize-tree-sitter-results/test-keywords_ts.json +++ b/extensions/vscode-colorize-tests/test/colorize-tree-sitter-results/test-keywords_ts.json @@ -85,7 +85,7 @@ }, { "c": "=>", - "t": "keyword.operator storage.type", + "t": "storage.type", "r": { "dark_plus": "storage.type: #569CD6", "light_plus": "storage.type: #0000FF", diff --git a/extensions/vscode-colorize-tests/test/colorize-tree-sitter-results/test_ts.json b/extensions/vscode-colorize-tests/test/colorize-tree-sitter-results/test_ts.json index 75a2bd87904..ca9e2cdd3a4 100644 --- a/extensions/vscode-colorize-tests/test/colorize-tree-sitter-results/test_ts.json +++ b/extensions/vscode-colorize-tests/test/colorize-tree-sitter-results/test_ts.json @@ -323,7 +323,7 @@ }, { "c": "constructor", - "t": "variable meta.definition.method entity.name.function storage.type", + "t": "variable meta.definition.method storage.type", "r": { "dark_plus": "storage.type: #569CD6", "light_plus": "storage.type: #0000FF", @@ -1457,7 +1457,7 @@ }, { "c": "constructor", - "t": "variable meta.definition.method entity.name.function storage.type", + "t": "variable meta.definition.method storage.type", "r": { "dark_plus": "storage.type: #569CD6", "light_plus": "storage.type: #0000FF", @@ -2717,7 +2717,7 @@ }, { "c": "=>", - "t": "keyword.operator storage.type", + "t": "storage.type", "r": { "dark_plus": "storage.type: #569CD6", "light_plus": "storage.type: #0000FF", @@ -3319,7 +3319,7 @@ }, { "c": "=>", - "t": "keyword.operator storage.type", + "t": "storage.type", "r": { "dark_plus": "storage.type: #569CD6", "light_plus": "storage.type: #0000FF", @@ -3879,7 +3879,7 @@ }, { "c": "=>", - "t": "keyword.operator storage.type", + "t": "storage.type", "r": { "dark_plus": "storage.type: #569CD6", "light_plus": "storage.type: #0000FF", diff --git a/src/vs/editor/common/languages/highlights/typescript.scm b/src/vs/editor/common/languages/highlights/typescript.scm index d6f52638659..5f6a6796883 100644 --- a/src/vs/editor/common/languages/highlights/typescript.scm +++ b/src/vs/editor/common/languages/highlights/typescript.scm @@ -61,9 +61,10 @@ (function_declaration name: (identifier) @entity.name.function) (method_definition - name: (property_identifier) @meta.definition.method @entity.name.function) + name: (property_identifier) @meta.definition.method @entity.name.function + (#not-eq? @entity.name.function "constructor")) (method_definition - name: (property_identifier) @storage.type + name: (property_identifier) @meta.definition.method @storage.type (#eq? @storage.type "constructor")) (method_signature name: (property_identifier) @meta.definition.method @entity.name.function) @@ -109,10 +110,6 @@ (predefined_type (["string" "boolean" "number" "any"])) @support.type.primitive (type_identifier) @entity.name.type -(("const") - (variable_declarator - name: (identifier) @variable.other.constant)) - ([ (identifier) (shorthand_property_identifier) @@ -192,7 +189,6 @@ "<<=" "==" "!=" - "=>" ">>" ">>=" ">>>" diff --git a/src/vs/workbench/services/themes/common/colorThemeData.ts b/src/vs/workbench/services/themes/common/colorThemeData.ts index c648e65d0a9..d6674cb5ab0 100644 --- a/src/vs/workbench/services/themes/common/colorThemeData.ts +++ b/src/vs/workbench/services/themes/common/colorThemeData.ts @@ -26,7 +26,6 @@ import { ThemeConfiguration } from './themeConfiguration.js'; import { ColorScheme } from '../../../../platform/theme/common/theme.js'; import { ColorId, FontStyle, MetadataConsts } from '../../../../editor/common/encodedTokenAttributes.js'; import { toStandardTokenType } from '../../../../editor/common/languages/supports/tokenization.js'; -import { findMatchingThemeRule } from '../../textMate/common/TMHelper.js'; const colorRegistry = Registry.as(ColorRegistryExtensions.ColorContribution); @@ -817,34 +816,25 @@ const defaultThemeColors: { [baseTheme: string]: ITextMateThemingRule[] } = { const noMatch = (_scope: ProbeScope) => -1; -function nameMatcher(identifers: string[], scope: ProbeScope): number { - function findInIdents(s: string, lastIndent: number): number { - for (let i = lastIndent - 1; i >= 0; i--) { - if (scopesAreMatching(s, identifers[i])) { - return i; - } - } +function nameMatcher(identifiers: string[], scopes: ProbeScope): number { + if (scopes.length < identifiers.length) { return -1; } - if (scope.length < identifers.length) { - return -1; - } - let lastScopeIndex = scope.length - 1; - let lastIdentifierIndex = findInIdents(scope[lastScopeIndex--], identifers.length); - if (lastIdentifierIndex >= 0) { - const score = (lastIdentifierIndex + 1) * 0x10000 + identifers[lastIdentifierIndex].length; - while (lastScopeIndex >= 0) { - lastIdentifierIndex = findInIdents(scope[lastScopeIndex--], lastIdentifierIndex); - if (lastIdentifierIndex === -1) { - return -1; + + let lastIndex = 0; + let score: number | undefined = undefined; + const every = identifiers.every((identifier, identifierIndex) => { + for (let i = lastIndex; i < scopes.length; i++) { + if (scopesAreMatching(scopes[i], identifier)) { + score = (identifierIndex + 1) * 0x10000 + identifier.length; + lastIndex = i + 1; + return true; } } - return score; - } - return -1; + return false; + }); + return every && score !== undefined ? score : -1; } - - function scopesAreMatching(thisScopeName: string, scopeName: string): boolean { if (!thisScopeName) { return false; @@ -906,18 +896,15 @@ export function findMetadata(colorThemeData: ColorThemeData, captureNames: strin metadata |= (languageId << MetadataConsts.LANGUAGEID_OFFSET); - const themeRule = findMatchingThemeRule(colorThemeData, captureNames); - let tokenStyle: TokenStyle | undefined; - if (!themeRule) { - tokenStyle = colorThemeData.resolveScopes(captureNames.map(name => [name]).reverse()); - } + const definitions: TextMateThemingRuleDefinitions = {}; + const tokenStyle = colorThemeData.resolveScopes([captureNames], definitions); if (captureNames.length > 0) { const standardToken = toStandardTokenType(captureNames[captureNames.length - 1]); metadata |= (standardToken << MetadataConsts.TOKEN_TYPE_OFFSET); } - switch (themeRule?.settings.fontStyle) { + switch (definitions.foreground?.settings.fontStyle) { case 'italic': metadata |= FontStyle.Italic | MetadataConsts.ITALIC_MASK; break; @@ -930,26 +917,9 @@ export function findMetadata(colorThemeData: ColorThemeData, captureNames: strin case 'strikethrough': metadata |= FontStyle.Strikethrough | MetadataConsts.STRIKETHROUGH_MASK; break; - default: - if (typeof tokenStyle?.italic !== 'undefined') { - const italicbit = (tokenStyle?.italic ? FontStyle.Italic : 0); - metadata |= italicbit | MetadataConsts.ITALIC_MASK; - } - if (typeof tokenStyle?.bold !== 'undefined') { - const boldBit = (tokenStyle?.bold ? FontStyle.Bold : 0); - metadata |= boldBit | MetadataConsts.BOLD_MASK; - } - if (typeof tokenStyle?.underline !== 'undefined') { - const underlineBit = (tokenStyle?.underline ? FontStyle.Underline : 0); - metadata |= underlineBit | MetadataConsts.UNDERLINE_MASK; - } - if (typeof tokenStyle?.strikethrough !== 'undefined') { - const strikethroughBit = (tokenStyle?.strikethrough ? FontStyle.Strikethrough : 0); - metadata |= strikethroughBit | MetadataConsts.STRIKETHROUGH_MASK; - } } - const foreground = themeRule ? themeRule.settings.foreground : tokenStyle?.foreground; - const tokenStyleForeground = foreground ? colorThemeData.getTokenColorIndex().get(foreground) : ColorId.DefaultForeground; + const foreground = tokenStyle?.foreground; + const tokenStyleForeground = (foreground !== undefined) ? colorThemeData.getTokenColorIndex().get(foreground) : ColorId.DefaultForeground; metadata |= tokenStyleForeground << MetadataConsts.FOREGROUND_OFFSET; return metadata; diff --git a/src/vs/workbench/services/themes/test/node/tokenStyleResolving.test.ts b/src/vs/workbench/services/themes/test/node/tokenStyleResolving.test.ts index 1eafdcff84f..13c3b9caf20 100644 --- a/src/vs/workbench/services/themes/test/node/tokenStyleResolving.test.ts +++ b/src/vs/workbench/services/themes/test/node/tokenStyleResolving.test.ts @@ -192,6 +192,9 @@ suite('Themes - TokenStyleResolving', () => { tokenStyle = themeData.resolveScopes([['meta.structure.dictionary.json', 'string.quoted.double.json']]); assertTokenStyle(tokenStyle, ts('#66D9EF', undefined), 'json property'); + tokenStyle = themeData.resolveScopes([['source.json', 'meta.structure.dictionary.json', 'string.quoted.double.json']]); + assertTokenStyle(tokenStyle, ts('#66D9EF', undefined), 'json property'); + tokenStyle = themeData.resolveScopes([['keyword'], ['storage.type'], ['entity.name.class']]); assertTokenStyle(tokenStyle, ts('#66D9EF', { italic: true, bold: false, underline: false }), 'storage.type'); diff --git a/src/vs/workbench/services/treeSitter/browser/treeSitterTokenizationFeature.ts b/src/vs/workbench/services/treeSitter/browser/treeSitterTokenizationFeature.ts index 9bd32453f1e..0829476700c 100644 --- a/src/vs/workbench/services/treeSitter/browser/treeSitterTokenizationFeature.ts +++ b/src/vs/workbench/services/treeSitter/browser/treeSitterTokenizationFeature.ts @@ -220,8 +220,8 @@ class TreeSitterTokenizationSupport extends Disposable implements ITreeSitterTok // Check that the current token doesn't just replace the last token if ((previousTokenStartOffset + currentTokenLength) === originalPreviousTokenEndOffset) { - // Current token and previous token span the exact same characters - endOffsetsAndScopes[tokenIndex - 1].scopes.push(capture.name); + // Current token and previous token span the exact same characters, replace the last scope + endOffsetsAndScopes[tokenIndex - 1].scopes[endOffsetsAndScopes[tokenIndex - 1].scopes.length - 1] = capture.name; } else { // The current token is within the previous token. Adjust the end of the previous token. endOffsetsAndScopes[tokenIndex - 1].endOffset = intermediateTokenOffset;