Improve unusable slowness of tree sitter syntax highlighting (#229951)

* Improve unusable slowness of tree sitter syntax highlighting
Part of #229935

* Improve improvement as discussed with Martin

* Fix score

* Fix test error

extensions/vscode-colorize-tests/test/colorize-tree-sitter-results/test-issue11_ts.json

@@ -1989,7 +1989,7 @@
 	},
 	{
 		"c": "=>",
-		"t": "keyword.operator storage.type",
+		"t": "storage.type",
 		"r": {
 			"dark_plus": "storage.type: #569CD6",
 			"light_plus": "storage.type: #0000FF",
@@ -2507,7 +2507,7 @@
 	},
 	{
 		"c": "=>",
-		"t": "keyword.operator storage.type",
+		"t": "storage.type",
 		"r": {
 			"dark_plus": "storage.type: #569CD6",
 			"light_plus": "storage.type: #0000FF",
@@ -2801,16 +2801,16 @@
 	},
 	{
 		"c": "t",
-		"t": "variable variable.other.constant",
+		"t": "variable",
 		"r": {
-			"dark_plus": "variable.other.constant: #4FC1FF",
-			"light_plus": "variable.other.constant: #0070C1",
+			"dark_plus": "variable: #9CDCFE",
+			"light_plus": "variable: #001080",
 			"dark_vs": "default: #D4D4D4",
 			"light_vs": "default: #000000",
 			"hc_black": "variable: #9CDCFE",
-			"dark_modern": "variable.other.constant: #4FC1FF",
-			"hc_light": "variable.other.constant: #02715D",
-			"light_modern": "variable.other.constant: #0070C1"
+			"dark_modern": "variable: #9CDCFE",
+			"hc_light": "variable: #001080",
+			"light_modern": "variable: #001080"
 		}
 	},
 	{

extensions/vscode-colorize-tests/test/colorize-tree-sitter-results/test-issue5431_ts.json

@@ -155,16 +155,16 @@
 	},
 	{
 		"c": "timeRange",
-		"t": "variable variable.other.constant",
+		"t": "variable",
 		"r": {
-			"dark_plus": "variable.other.constant: #4FC1FF",
-			"light_plus": "variable.other.constant: #0070C1",
+			"dark_plus": "variable: #9CDCFE",
+			"light_plus": "variable: #001080",
 			"dark_vs": "default: #D4D4D4",
 			"light_vs": "default: #000000",
 			"hc_black": "variable: #9CDCFE",
-			"dark_modern": "variable.other.constant: #4FC1FF",
-			"hc_light": "variable.other.constant: #02715D",
-			"light_modern": "variable.other.constant: #0070C1"
+			"dark_modern": "variable: #9CDCFE",
+			"hc_light": "variable: #001080",
+			"light_modern": "variable: #001080"
 		}
 	},
 	{

extensions/vscode-colorize-tests/test/colorize-tree-sitter-results/test-issue5566_ts.json

@@ -85,16 +85,16 @@
 	},
 	{
 		"c": "foo",
-		"t": "variable entity.name.function variable.other.constant",
+		"t": "variable entity.name.function",
 		"r": {
-			"dark_plus": "variable.other.constant: #4FC1FF",
-			"light_plus": "variable.other.constant: #0070C1",
+			"dark_plus": "entity.name.function: #DCDCAA",
+			"light_plus": "entity.name.function: #795E26",
 			"dark_vs": "default: #D4D4D4",
 			"light_vs": "default: #000000",
-			"hc_black": "variable: #9CDCFE",
-			"dark_modern": "variable.other.constant: #4FC1FF",
-			"hc_light": "variable.other.constant: #02715D",
-			"light_modern": "variable.other.constant: #0070C1"
+			"hc_black": "entity.name.function: #DCDCAA",
+			"dark_modern": "entity.name.function: #DCDCAA",
+			"hc_light": "entity.name.function: #5E2CBC",
+			"light_modern": "entity.name.function: #795E26"
 		}
 	},
 	{
@@ -169,7 +169,7 @@
 	},
 	{
 		"c": "=>",
-		"t": "keyword.operator storage.type",
+		"t": "storage.type",
 		"r": {
 			"dark_plus": "storage.type: #569CD6",
 			"light_plus": "storage.type: #0000FF",

extensions/vscode-colorize-tests/test/colorize-tree-sitter-results/test-keywords_ts.json

@@ -85,7 +85,7 @@
 	},
 	{
 		"c": "=>",
-		"t": "keyword.operator storage.type",
+		"t": "storage.type",
 		"r": {
 			"dark_plus": "storage.type: #569CD6",
 			"light_plus": "storage.type: #0000FF",

extensions/vscode-colorize-tests/test/colorize-tree-sitter-results/test_ts.json

@@ -323,7 +323,7 @@
 	},
 	{
 		"c": "constructor",
-		"t": "variable meta.definition.method entity.name.function storage.type",
+		"t": "variable meta.definition.method storage.type",
 		"r": {
 			"dark_plus": "storage.type: #569CD6",
 			"light_plus": "storage.type: #0000FF",
@@ -1457,7 +1457,7 @@
 	},
 	{
 		"c": "constructor",
-		"t": "variable meta.definition.method entity.name.function storage.type",
+		"t": "variable meta.definition.method storage.type",
 		"r": {
 			"dark_plus": "storage.type: #569CD6",
 			"light_plus": "storage.type: #0000FF",
@@ -2717,7 +2717,7 @@
 	},
 	{
 		"c": "=>",
-		"t": "keyword.operator storage.type",
+		"t": "storage.type",
 		"r": {
 			"dark_plus": "storage.type: #569CD6",
 			"light_plus": "storage.type: #0000FF",
@@ -3319,7 +3319,7 @@
 	},
 	{
 		"c": "=>",
-		"t": "keyword.operator storage.type",
+		"t": "storage.type",
 		"r": {
 			"dark_plus": "storage.type: #569CD6",
 			"light_plus": "storage.type: #0000FF",
@@ -3879,7 +3879,7 @@
 	},
 	{
 		"c": "=>",
-		"t": "keyword.operator storage.type",
+		"t": "storage.type",
 		"r": {
 			"dark_plus": "storage.type: #569CD6",
 			"light_plus": "storage.type: #0000FF",

src/vs/editor/common/languages/highlights/typescript.scm

@@ -61,9 +61,10 @@
 (function_declaration
   name: (identifier) @entity.name.function)
 (method_definition
-  name: (property_identifier) @meta.definition.method @entity.name.function)
+  name: (property_identifier) @meta.definition.method @entity.name.function
+  (#not-eq? @entity.name.function "constructor"))
 (method_definition
-  name: (property_identifier) @storage.type
+  name: (property_identifier) @meta.definition.method @storage.type
   (#eq? @storage.type "constructor"))
 (method_signature
   name: (property_identifier) @meta.definition.method @entity.name.function)
@@ -109,10 +110,6 @@
 (predefined_type (["string" "boolean" "number" "any"])) @support.type.primitive
 (type_identifier) @entity.name.type
 
-(("const")
-  (variable_declarator
-  	name: (identifier) @variable.other.constant))
-
 ([
   (identifier)
   (shorthand_property_identifier)
@@ -192,7 +189,6 @@
   "<<="
   "=="
   "!="
-  "=>"
   ">>"
   ">>="
   ">>>"

src/vs/workbench/services/themes/common/colorThemeData.ts

@@ -26,7 +26,6 @@ import { ThemeConfiguration } from './themeConfiguration.js';
 import { ColorScheme } from '../../../../platform/theme/common/theme.js';
 import { ColorId, FontStyle, MetadataConsts } from '../../../../editor/common/encodedTokenAttributes.js';
 import { toStandardTokenType } from '../../../../editor/common/languages/supports/tokenization.js';
-import { findMatchingThemeRule } from '../../textMate/common/TMHelper.js';
 
 const colorRegistry = Registry.as<IColorRegistry>(ColorRegistryExtensions.ColorContribution);
 
@@ -817,34 +816,25 @@ const defaultThemeColors: { [baseTheme: string]: ITextMateThemingRule[] } = {
 
 const noMatch = (_scope: ProbeScope) => -1;
 
-function nameMatcher(identifers: string[], scope: ProbeScope): number {
-	function findInIdents(s: string, lastIndent: number): number {
-		for (let i = lastIndent - 1; i >= 0; i--) {
-			if (scopesAreMatching(s, identifers[i])) {
-				return i;
-			}
-		}
+function nameMatcher(identifiers: string[], scopes: ProbeScope): number {
+	if (scopes.length < identifiers.length) {
 		return -1;
 	}
-	if (scope.length < identifers.length) {
-		return -1;
-	}
-	let lastScopeIndex = scope.length - 1;
-	let lastIdentifierIndex = findInIdents(scope[lastScopeIndex--], identifers.length);
-	if (lastIdentifierIndex >= 0) {
-		const score = (lastIdentifierIndex + 1) * 0x10000 + identifers[lastIdentifierIndex].length;
-		while (lastScopeIndex >= 0) {
-			lastIdentifierIndex = findInIdents(scope[lastScopeIndex--], lastIdentifierIndex);
-			if (lastIdentifierIndex === -1) {
-				return -1;
+
+	let lastIndex = 0;
+	let score: number | undefined = undefined;
+	const every = identifiers.every((identifier, identifierIndex) => {
+		for (let i = lastIndex; i < scopes.length; i++) {
+			if (scopesAreMatching(scopes[i], identifier)) {
+				score = (identifierIndex + 1) * 0x10000 + identifier.length;
+				lastIndex = i + 1;
+				return true;
 			}
 		}
-		return score;
-	}
-	return -1;
+		return false;
+	});
+	return every && score !== undefined ? score : -1;
 }
-
-
 function scopesAreMatching(thisScopeName: string, scopeName: string): boolean {
 	if (!thisScopeName) {
 		return false;
@@ -906,18 +896,15 @@ export function findMetadata(colorThemeData: ColorThemeData, captureNames: strin
 
 	metadata |= (languageId << MetadataConsts.LANGUAGEID_OFFSET);
 
-	const themeRule = findMatchingThemeRule(colorThemeData, captureNames);
-	let tokenStyle: TokenStyle | undefined;
-	if (!themeRule) {
-		tokenStyle = colorThemeData.resolveScopes(captureNames.map(name => [name]).reverse());
-	}
+	const definitions: TextMateThemingRuleDefinitions = {};
+	const tokenStyle = colorThemeData.resolveScopes([captureNames], definitions);
 
 	if (captureNames.length > 0) {
 		const standardToken = toStandardTokenType(captureNames[captureNames.length - 1]);
 		metadata |= (standardToken << MetadataConsts.TOKEN_TYPE_OFFSET);
 	}
 
-	switch (themeRule?.settings.fontStyle) {
+	switch (definitions.foreground?.settings.fontStyle) {
 		case 'italic':
 			metadata |= FontStyle.Italic | MetadataConsts.ITALIC_MASK;
 			break;
@@ -930,26 +917,9 @@ export function findMetadata(colorThemeData: ColorThemeData, captureNames: strin
 		case 'strikethrough':
 			metadata |= FontStyle.Strikethrough | MetadataConsts.STRIKETHROUGH_MASK;
 			break;
-		default:
-			if (typeof tokenStyle?.italic !== 'undefined') {
-				const italicbit = (tokenStyle?.italic ? FontStyle.Italic : 0);
-				metadata |= italicbit | MetadataConsts.ITALIC_MASK;
-			}
-			if (typeof tokenStyle?.bold !== 'undefined') {
-				const boldBit = (tokenStyle?.bold ? FontStyle.Bold : 0);
-				metadata |= boldBit | MetadataConsts.BOLD_MASK;
-			}
-			if (typeof tokenStyle?.underline !== 'undefined') {
-				const underlineBit = (tokenStyle?.underline ? FontStyle.Underline : 0);
-				metadata |= underlineBit | MetadataConsts.UNDERLINE_MASK;
-			}
-			if (typeof tokenStyle?.strikethrough !== 'undefined') {
-				const strikethroughBit = (tokenStyle?.strikethrough ? FontStyle.Strikethrough : 0);
-				metadata |= strikethroughBit | MetadataConsts.STRIKETHROUGH_MASK;
-			}
 	}
-	const foreground = themeRule ? themeRule.settings.foreground : tokenStyle?.foreground;
-	const tokenStyleForeground = foreground ? colorThemeData.getTokenColorIndex().get(foreground) : ColorId.DefaultForeground;
+	const foreground = tokenStyle?.foreground;
+	const tokenStyleForeground = (foreground !== undefined) ? colorThemeData.getTokenColorIndex().get(foreground) : ColorId.DefaultForeground;
 	metadata |= tokenStyleForeground << MetadataConsts.FOREGROUND_OFFSET;
 
 	return metadata;

src/vs/workbench/services/themes/test/node/tokenStyleResolving.test.ts

@@ -192,6 +192,9 @@ suite('Themes - TokenStyleResolving', () => {
 		tokenStyle = themeData.resolveScopes([['meta.structure.dictionary.json', 'string.quoted.double.json']]);
 		assertTokenStyle(tokenStyle, ts('#66D9EF', undefined), 'json property');
 
+		tokenStyle = themeData.resolveScopes([['source.json', 'meta.structure.dictionary.json', 'string.quoted.double.json']]);
+		assertTokenStyle(tokenStyle, ts('#66D9EF', undefined), 'json property');
+
 		tokenStyle = themeData.resolveScopes([['keyword'], ['storage.type'], ['entity.name.class']]);
 		assertTokenStyle(tokenStyle, ts('#66D9EF', { italic: true, bold: false, underline: false }), 'storage.type');
 

src/vs/workbench/services/treeSitter/browser/treeSitterTokenizationFeature.ts

@@ -220,8 +220,8 @@ class TreeSitterTokenizationSupport extends Disposable implements ITreeSitterTok
 
 				// Check that the current token doesn't just replace the last token
 				if ((previousTokenStartOffset + currentTokenLength) === originalPreviousTokenEndOffset) {
-					// Current token and previous token span the exact same characters
-					endOffsetsAndScopes[tokenIndex - 1].scopes.push(capture.name);
+					// Current token and previous token span the exact same characters, replace the last scope
+					endOffsetsAndScopes[tokenIndex - 1].scopes[endOffsetsAndScopes[tokenIndex - 1].scopes.length - 1] = capture.name;
 				} else {
 					// The current token is within the previous token. Adjust the end of the previous token.
 					endOffsetsAndScopes[tokenIndex - 1].endOffset = intermediateTokenOffset;