microsoft
/
vscode-textmate
зеркало из https://github.com/microsoft/vscode-textmate.git
1
0
Форкнуть 0
Код Задачи Пакеты Проекты Релизы Вики Активность

Renames TemporaryStandardTokenType to OptionalStandardTokenType. 

Renames Other to NotSet and fixes semantics.
This commit is contained in:
Henning Dieterichs 2022-04-04 14:47:14 +02:00
Родитель a22e778360
Коммит d6cd3eb508
Не найден ключ, соответствующий данной подписи
Идентификатор ключа GPG: 771381EFFDB9EC06
2 изменённых файлов: 49 добавлений и 45 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

64
src/grammar.ts
Просмотреть файл

@ -20,12 +20,14 @@ const performanceNow = (function () {
}
})();
export const enum TemporaryStandardTokenType {
// Must have the same values as `StandardTokenType`!
export const enum OptionalStandardTokenType {
Other = 0,
Comment = 1,
String = 2,
RegEx = 3,
MetaEmbedded = 8
// Indicates that no token type is set.
NotSet = 8
}
export function createGrammar(scopeName: string, grammar: IRawGrammar, initialLanguage: number, embeddedLanguages: IEmbeddedLanguagesMap | null, tokenTypes: ITokenTypeMap | null, grammarRepository: IGrammarRepository & IThemeProvider, onigLib: IOnigLib): Grammar {
@ -294,10 +296,10 @@ function collectInjections(result: Injection[], selector: string, rule: IRawRule
export class ScopeMetadata {
public readonly scopeName: string;
public readonly languageId: number;
public readonly tokenType: TemporaryStandardTokenType;
public readonly tokenType: OptionalStandardTokenType;
public readonly themeData: ThemeTrieElementRule[] | null;
constructor(scopeName: string, languageId: number, tokenType: TemporaryStandardTokenType, themeData: ThemeTrieElementRule[] | null) {
constructor(scopeName: string, languageId: number, tokenType: OptionalStandardTokenType, themeData: ThemeTrieElementRule[] | null) {
this.scopeName = scopeName;
this.languageId = languageId;
this.tokenType = tokenType;
@ -321,7 +323,7 @@ class ScopeMetadataProvider {
this._defaultMetaData = new ScopeMetadata(
'',
this._initialLanguage,
TemporaryStandardTokenType.Other,
OptionalStandardTokenType.NotSet,
[this._themeProvider.getDefaults()]
);
@ -360,7 +362,7 @@ class ScopeMetadataProvider {
this._defaultMetaData = new ScopeMetadata(
'',
this._initialLanguage,
TemporaryStandardTokenType.Other,
OptionalStandardTokenType.NotSet,
[this._themeProvider.getDefaults()]
);
}
@ -425,20 +427,20 @@ class ScopeMetadataProvider {
}
private static STANDARD_TOKEN_TYPE_REGEXP = /\b(comment|string|regex|meta\.embedded)\b/;
private _toStandardTokenType(tokenType: string): TemporaryStandardTokenType {
private _toStandardTokenType(tokenType: string): OptionalStandardTokenType {
const m = tokenType.match(ScopeMetadataProvider.STANDARD_TOKEN_TYPE_REGEXP);
if (!m) {
return TemporaryStandardTokenType.Other;
return OptionalStandardTokenType.NotSet;
}
switch (m[1]) {
case 'comment':
return TemporaryStandardTokenType.Comment;
return OptionalStandardTokenType.Comment;
case 'string':
return TemporaryStandardTokenType.String;
return OptionalStandardTokenType.String;
case 'regex':
return TemporaryStandardTokenType.RegEx;
return OptionalStandardTokenType.RegEx;
case 'meta.embedded':
return TemporaryStandardTokenType.MetaEmbedded;
return OptionalStandardTokenType.Other;
}
throw new Error('Unexpected match for standard token type!');
}
@ -1225,7 +1227,7 @@ export class StackElementMetadata {
return (metadata & MetadataConsts.LANGUAGEID_MASK) >>> MetadataConsts.LANGUAGEID_OFFSET;
}
public static getTokenType(metadata: number): number {
public static getTokenType(metadata: number): StandardTokenType {
return (metadata & MetadataConsts.TOKEN_TYPE_MASK) >>> MetadataConsts.TOKEN_TYPE_OFFSET;
}
@ -1241,7 +1243,11 @@ export class StackElementMetadata {
return (metadata & MetadataConsts.BACKGROUND_MASK) >>> MetadataConsts.BACKGROUND_OFFSET;
}
public static set(metadata: number, languageId: number, tokenType: TemporaryStandardTokenType, fontStyle: FontStyle, foreground: number, background: number): number {
/**
* Updates the fields in `metadata`.
* A value of `0` or `NotSet` indicates that the corresponding field should be left as is.
*/
public static set(metadata: number, languageId: number, tokenType: OptionalStandardTokenType, fontStyle: FontStyle, foreground: number, background: number): number {
let _languageId = StackElementMetadata.getLanguageId(metadata);
let _tokenType = StackElementMetadata.getTokenType(metadata);
let _fontStyle = StackElementMetadata.getFontStyle(metadata);
@ -1251,8 +1257,8 @@ export class StackElementMetadata {
if (languageId !== 0) {
_languageId = languageId;
}
if (tokenType !== TemporaryStandardTokenType.Other) {
_tokenType = tokenType === TemporaryStandardTokenType.MetaEmbedded ? StandardTokenType.Other : tokenType;
if (tokenType !== OptionalStandardTokenType.NotSet) {
_tokenType = fromOptionalTokenType(tokenType);
}
if (fontStyle !== FontStyle.NotSet) {
_fontStyle = fontStyle;
@ -1669,7 +1675,7 @@ class LineTokens {
const scopes = scopesList.generateScopes();
for (const tokenType of this._tokenTypeOverrides) {
if (tokenType.matcher(scopes)) {
metadata = StackElementMetadata.set(metadata, 0, toTemporaryType(tokenType.type), FontStyle.NotSet, 0, 0);
metadata = StackElementMetadata.set(metadata, 0, toOptionalTokenType(tokenType.type), FontStyle.NotSet, 0, 0);
}
}
}
@ -1751,18 +1757,16 @@ class LineTokens {
}
}
function toTemporaryType(standardType: StandardTokenType): TemporaryStandardTokenType {
switch (standardType) {
case StandardTokenType.RegEx:
return TemporaryStandardTokenType.RegEx;
case StandardTokenType.String:
return TemporaryStandardTokenType.String;
case StandardTokenType.Comment:
return TemporaryStandardTokenType.Comment;
case StandardTokenType.Other:
default:
// `MetaEmbedded` is the same scope as `Other`
// but it overwrites existing token types in the stack.
return TemporaryStandardTokenType.MetaEmbedded;
function toOptionalTokenType(standardType: StandardTokenType): OptionalStandardTokenType {
return standardType as any as OptionalStandardTokenType;
}
function fromOptionalTokenType(
standardType:
| OptionalStandardTokenType.Other
| OptionalStandardTokenType.Comment
| OptionalStandardTokenType.String
| OptionalStandardTokenType.RegEx
): StandardTokenType {
return standardType as any as StandardTokenType;
}

28
src/tests/grammar.test.ts
Просмотреть файл

@ -4,7 +4,7 @@
import * as assert from 'assert';
import { StandardTokenType } from '../main';
import { StackElementMetadata, TemporaryStandardTokenType } from '../grammar';
import { StackElementMetadata, OptionalStandardTokenType } from '../grammar';
import { FontStyle } from '../theme';
function assertEquals(metadata: number, languageId: number, tokenType: StandardTokenType, fontStyle: FontStyle, foreground: number, background: number): void {
@ -28,55 +28,55 @@ function assertEquals(metadata: number, languageId: number, tokenType: StandardT
}
test('StackElementMetadata works', () => {
let value = StackElementMetadata.set(0, 1, TemporaryStandardTokenType.RegEx, FontStyle.Underline | FontStyle.Bold, 101, 102);
let value = StackElementMetadata.set(0, 1, OptionalStandardTokenType.RegEx, FontStyle.Underline | FontStyle.Bold, 101, 102);
assertEquals(value, 1, StandardTokenType.RegEx, FontStyle.Underline | FontStyle.Bold, 101, 102);
});
test('StackElementMetadata can overwrite languageId', () => {
let value = StackElementMetadata.set(0, 1, TemporaryStandardTokenType.RegEx, FontStyle.Underline | FontStyle.Bold, 101, 102);
let value = StackElementMetadata.set(0, 1, OptionalStandardTokenType.RegEx, FontStyle.Underline | FontStyle.Bold, 101, 102);
assertEquals(value, 1, StandardTokenType.RegEx, FontStyle.Underline | FontStyle.Bold, 101, 102);
value = StackElementMetadata.set(value, 2, TemporaryStandardTokenType.Other, FontStyle.NotSet, 0, 0);
value = StackElementMetadata.set(value, 2, OptionalStandardTokenType.NotSet, FontStyle.NotSet, 0, 0);
assertEquals(value, 2, StandardTokenType.RegEx, FontStyle.Underline | FontStyle.Bold, 101, 102);
});
test('StackElementMetadata can overwrite tokenType', () => {
let value = StackElementMetadata.set(0, 1, TemporaryStandardTokenType.RegEx, FontStyle.Underline | FontStyle.Bold, 101, 102);
let value = StackElementMetadata.set(0, 1, OptionalStandardTokenType.RegEx, FontStyle.Underline | FontStyle.Bold, 101, 102);
assertEquals(value, 1, StandardTokenType.RegEx, FontStyle.Underline | FontStyle.Bold, 101, 102);
value = StackElementMetadata.set(value, 0, TemporaryStandardTokenType.Comment, FontStyle.NotSet, 0, 0);
value = StackElementMetadata.set(value, 0, OptionalStandardTokenType.Comment, FontStyle.NotSet, 0, 0);
assertEquals(value, 1, StandardTokenType.Comment, FontStyle.Underline | FontStyle.Bold, 101, 102);
});
test('StackElementMetadata can overwrite font style', () => {
let value = StackElementMetadata.set(0, 1, TemporaryStandardTokenType.RegEx, FontStyle.Underline | FontStyle.Bold, 101, 102);
let value = StackElementMetadata.set(0, 1, OptionalStandardTokenType.RegEx, FontStyle.Underline | FontStyle.Bold, 101, 102);
assertEquals(value, 1, StandardTokenType.RegEx, FontStyle.Underline | FontStyle.Bold, 101, 102);
value = StackElementMetadata.set(value, 0, TemporaryStandardTokenType.Other, FontStyle.None, 0, 0);
value = StackElementMetadata.set(value, 0, OptionalStandardTokenType.NotSet, FontStyle.None, 0, 0);
assertEquals(value, 1, StandardTokenType.RegEx, FontStyle.None, 101, 102);
});
test('StackElementMetadata can overwrite font style with strikethrough', () => {
let value = StackElementMetadata.set(0, 1, TemporaryStandardTokenType.RegEx, FontStyle.Strikethrough, 101, 102);
let value = StackElementMetadata.set(0, 1, OptionalStandardTokenType.RegEx, FontStyle.Strikethrough, 101, 102);
assertEquals(value, 1, StandardTokenType.RegEx, FontStyle.Strikethrough, 101, 102);
value = StackElementMetadata.set(value, 0, TemporaryStandardTokenType.Other, FontStyle.None, 0, 0);
value = StackElementMetadata.set(value, 0, OptionalStandardTokenType.NotSet, FontStyle.None, 0, 0);
assertEquals(value, 1, StandardTokenType.RegEx, FontStyle.None, 101, 102);
});
test('StackElementMetadata can overwrite foreground', () => {
let value = StackElementMetadata.set(0, 1, TemporaryStandardTokenType.RegEx, FontStyle.Underline | FontStyle.Bold, 101, 102);
let value = StackElementMetadata.set(0, 1, OptionalStandardTokenType.RegEx, FontStyle.Underline | FontStyle.Bold, 101, 102);
assertEquals(value, 1, StandardTokenType.RegEx, FontStyle.Underline | FontStyle.Bold, 101, 102);
value = StackElementMetadata.set(value, 0, TemporaryStandardTokenType.Other, FontStyle.NotSet, 5, 0);
value = StackElementMetadata.set(value, 0, OptionalStandardTokenType.NotSet, FontStyle.NotSet, 5, 0);
assertEquals(value, 1, StandardTokenType.RegEx, FontStyle.Underline | FontStyle.Bold, 5, 102);
});
test('StackElementMetadata can overwrite background', () => {
let value = StackElementMetadata.set(0, 1, TemporaryStandardTokenType.RegEx, FontStyle.Underline | FontStyle.Bold, 101, 102);
let value = StackElementMetadata.set(0, 1, OptionalStandardTokenType.RegEx, FontStyle.Underline | FontStyle.Bold, 101, 102);
assertEquals(value, 1, StandardTokenType.RegEx, FontStyle.Underline | FontStyle.Bold, 101, 102);
value = StackElementMetadata.set(value, 0, TemporaryStandardTokenType.Other, FontStyle.NotSet, 0, 7);
value = StackElementMetadata.set(value, 0, OptionalStandardTokenType.NotSet, FontStyle.NotSet, 0, 7);
assertEquals(value, 1, StandardTokenType.RegEx, FontStyle.Underline | FontStyle.Bold, 101, 7);
});