microsoft
/
vscode-textmate
зеркало из https://github.com/microsoft/vscode-textmate.git
1
0
Форкнуть 0
Код Задачи Пакеты Проекты Релизы Вики Активность

Children inherit parentScopes

This commit is contained in:
Alex Dima 2016-11-25 14:00:12 -08:00
Родитель a29f38bf32
Коммит 15debdd1c9
6 изменённых файлов: 246 добавлений и 102 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

168
src/tests/themeTest.ts
Просмотреть файл

@ -6,7 +6,7 @@
import * as fs from 'fs';
import * as path from 'path';
import { IEmbeddedLanguagesMap } from '../main';
import { tokenizeWithTheme, IThemedToken } from './themedTokenizer';
import { tokenizeWithTheme, IThemedToken, IThemedTokenScopeExplanation } from './themedTokenizer';
import { Resolver, ThemeData } from './themes.test';
interface IExpected {
@ -69,7 +69,6 @@ export class ThemeTest {
const testFileExpected = ThemeTest._readJSONFile<IExpected>(EXPECTED_FILE_PATH);
const EXPECTED_PATCH_FILE_PATH = path.join(THEMES_TEST_PATH, 'tests', testFile + '.result.patch');
console.log(EXPECTED_PATCH_FILE_PATH);
const testFileExpectedPatch = ThemeTest._readJSONFile<IExpectedPatch>(EXPECTED_PATCH_FILE_PATH);
// Determine the language
@ -142,7 +141,8 @@ export class ThemeTest {
public hasDiff(): boolean {
for (let i = 0; i < this.tests.length; i++) {
if (this.tests[i].patchedDiff.length > 0) {
let test = this.tests[i];
if (test.patchedDiff && test.patchedDiff.length > 0) {
return true;
}
}
@ -162,10 +162,22 @@ export class ThemeTest {
}
}
interface IActualCanonicalToken {
content: string;
color: string;
scopes: IThemedTokenScopeExplanation[];
}
interface IExpectedCanonicalToken {
oldIndex: number;
content: string;
color: string;
_r: string;
_t: string;
}
interface ITokenizationDiff {
oldIndex: number;
oldToken: IExpectedTokenization;
newToken: IThemedToken;
newToken: IActualCanonicalToken;
}
interface IDiffPageData {
@ -215,16 +227,46 @@ class SingleThemeTest {
this.expected = expected;
this.expectedPatch = expectedPatch;
this.patchedExpected = this.expected.slice(0);
for (let i = 0; i < this.expectedPatch.length; i++) {
let patch = this.expectedPatch[i];
this.patchedExpected = [];
let patchIndex = this.expectedPatch.length - 1;
for (let i = this.expected.length - 1; i >= 0; i--) {
let expectedElement = this.expected[i];
let content = expectedElement.content;
while (patchIndex >= 0 && i === this.expectedPatch[patchIndex].index) {
let patch = this.expectedPatch[patchIndex];
this.patchedExpected[patch.index] = {
_r: this.patchedExpected[patch.index]._r,
_t: this.patchedExpected[patch.index]._t,
let patchContentIndex = content.lastIndexOf(patch.content);
let afterContent = content.substr(patchContentIndex + patch.content.length);
if (afterContent.length > 0) {
this.patchedExpected.unshift({
_r: expectedElement._r,
_t: expectedElement._t,
content: afterContent,
color: expectedElement.color
});
}
this.patchedExpected.unshift({
_r: expectedElement._r,
_t: expectedElement._t,
content: patch.content,
color: patch.newColor
};
});
content = content.substr(0, patchContentIndex);
patchIndex--;
}
if (content.length > 0) {
this.patchedExpected.unshift({
_r: expectedElement._r,
_t: expectedElement._t,
content: content,
color: expectedElement.color
});
}
}
this.backgroundColor = null;
@ -272,67 +314,69 @@ class SingleThemeTest {
});
}
private static computeThemeTokenizationDiff(actual: IThemedToken[], expected: IExpectedTokenization[]): ITokenizationDiff[] {
let diffs: ITokenizationDiff[] = [];
let i = 0, j = 0, len = actual.length, lenJ = expected.length;
do {
if (i >= len && j >= lenJ) {
// ok
break;
private static computeThemeTokenizationDiff(_actual: IThemedToken[], _expected: IExpectedTokenization[]): ITokenizationDiff[] {
let canonicalTokens: string[] = [];
for (let i = 0, len = _actual.length; i < len; i++) {
let explanation = _actual[i].explanation;
for (let j = 0, lenJ = explanation.length; j < lenJ; j++) {
canonicalTokens.push(explanation[j].content);
}
}
if (i >= len) {
// will fail
throw new Error('Reached end of actual before end of expected');
}
let actual: IActualCanonicalToken[] = [];
for (let i = 0, len = _actual.length; i < len; i++) {
let item = _actual[i];
if (j >= lenJ) {
// will fail
throw new Error('Reached end of expected before end of actual');
}
let actualContent = actual[i].content;
let actualColor = actual[i].color;
if (actualColor.length > 7) {
// TODO: remove alpha to match expected tests format
actualColor = actualColor.substring(0, 7);
}
while (actualContent.length > 0 && j < lenJ) {
let expectedContent = expected[j].content;
let expectedColor = expected[j].color;
let contentIsInvisible = /^\s+$/.test(expectedContent);
if (!contentIsInvisible && actualColor !== expectedColor) {
// console.log('COLOR MISMATCH: ', actualColor, expectedColor);
// select the same token from the explanation
let reducedExplanation = actual[i].explanation.filter((e) => e.content === expectedContent);
if (reducedExplanation.length === 0) {
reducedExplanation = actual[i].explanation;
}
diffs.push({
oldIndex: j,
oldToken: expected[j],
newToken: {
content: actual[i].content,
color: actual[i].color,
explanation: reducedExplanation
}
for (let j = 0, lenJ = item.explanation.length; j < lenJ; j++) {
actual.push({
content: item.explanation[j].content,
color: item.color,
scopes: item.explanation[j].scopes
});
}
if (actualContent.substr(0, expectedContent.length) !== expectedContent) {
throw new Error(`at ${actualContent} (${i}-${j}), content mismatch: ${actualContent}, ${expectedContent}`);
}
actualContent = actualContent.substr(expectedContent.length);
let expected: IExpectedCanonicalToken[] = [];
for (let i = 0, len = _expected.length, canonicalIndex = 0; i < len; i++) {
let item = _expected[i];
j++;
let content = item.content;
while (content.length > 0) {
expected.push({
oldIndex: i,
content: canonicalTokens[canonicalIndex],
color: item.color,
_t: item._t,
_r: item._r
});
content = content.substr(canonicalTokens[canonicalIndex].length);
canonicalIndex++;
}
}
i++;
} while (true);
if (actual.length !== expected.length) {
throw new Error('Content mismatch');
}
let diffs: ITokenizationDiff[] = [];
for (let i = 0, len = actual.length; i < len; i++) {
let expectedItem = expected[i];
let actualItem = actual[i];
let contentIsInvisible = /^\s+$/.test(expectedItem.content);
if (contentIsInvisible) {
continue;
}
if (actualItem.color.substr(0, 7) !== expectedItem.color) {
diffs.push({
oldIndex: expectedItem.oldIndex,
oldToken: expectedItem,
newToken: actualItem
});
}
}
return diffs;
}

50
src/tests/themes.test.ts
Просмотреть файл

@ -7,6 +7,7 @@ import * as fs from 'fs';
import * as path from 'path';
import * as assert from 'assert';
import { Registry, RegistryOptions, IRawTheme } from '../main';
import { ScopeListElement, ScopeMetadata, StackElementMetadata } from '../grammar';
import {
Theme, strcmp, strArrCmp, ThemeTrieElement, ThemeTrieElementRule,
parseTheme, ParsedThemeRule, FontStyle, ColorMap
@ -111,7 +112,7 @@ export class Resolver implements RegistryOptions {
return path.join(THEMES_TEST_PATH, grammar.path);
}
}
console.warn('missing gramamr for ' + scopeName);
// console.warn('missing grammar for ' + scopeName);
}
}
@ -165,12 +166,8 @@ class ThemeInfo {
function assertThemeTest(test: ThemeTest, themeDatas: ThemeData[]): void {
it(test.testName, (done) => {
test.evaluate(themeDatas, (err) => {
console.log('HERE I AM!!');
test.writeDiffPage();
assert.ok(!test.hasDiff(), 'no more unpatched differences');
done();
});
}).timeout(5000);
@ -217,6 +214,49 @@ function assertThemeTest(test: ThemeTest, themeDatas: ThemeData[]): void {
})();
describe('Theme matching', () => {
it('gives higher priority to parent matches 1', () => {
let theme = Theme.createFromRawTheme({
settings: [
{ settings: { foreground: '#100000', background: '#200000' } },
{ scope: 'c a', settings: { foreground: '#300000' } },
{ scope: 'd a.b', settings: { foreground: '#400000' } },
{ scope: 'a', settings: { foreground: '#500000' } },
]
});
let colorMap = new ColorMap();
const _NOT_SET = 0;
const _A = colorMap.getId('#100000');
const _B = colorMap.getId('#200000');
const _C = colorMap.getId('#500000');
const _D = colorMap.getId('#300000');
const _E = colorMap.getId('#400000');
let actual = theme.match('a.b');
assert.deepEqual(actual, [
new ThemeTrieElementRule(['d'], FontStyle.NotSet, _E, _NOT_SET),
new ThemeTrieElementRule(['c'], FontStyle.NotSet, _D, _NOT_SET),
new ThemeTrieElementRule(null, FontStyle.NotSet, _C, _NOT_SET),
]);
});
it('gives higher priority to parent matches 3', () => {
let theme = Theme.createFromRawTheme({
settings: [
{ settings: { foreground: '#100000', background: '#200000' } },
{ scope: 'meta.tag entity', settings: { foreground: '#300000' } },
{ scope: 'meta.selector.css entity.name.tag', settings: { foreground: '#400000' } },
{ scope: 'entity', settings: { foreground: '#500000' } },
]
});
let root = new ScopeListElement(null, 'text.html.cshtml', 0);
let parent = new ScopeListElement(root, 'meta.tag.structure.any.html', 0);
let r = ScopeListElement.mergeMetadata(0, parent, new ScopeMetadata('entity.name.tag.structure.any.html', 0, 0, theme.match('entity.name.tag.structure.any.html')));
let colorMap = theme.getColorMap();
assert.equal(colorMap[StackElementMetadata.getForeground(r)], '#300000');
});
it('can match', () => {
let theme = Theme.createFromRawTheme({
settings: [

11
src/theme.ts
Просмотреть файл

@ -299,6 +299,14 @@ export class ThemeTrieElementRule {
return new ThemeTrieElementRule(this.parentScopes, this.fontStyle, this.foreground, this.background);
}
public static cloneArr(arr:ThemeTrieElementRule[]): ThemeTrieElementRule[] {
let r: ThemeTrieElementRule[] = [];
for (let i = 0, len = arr.length; i < len; i++) {
r[i] = arr[i].clone();
}
return r;
}
public acceptOverwrite(fontStyle: number, foreground: number, background: number): void {
if (fontStyle !== FontStyle.NotSet) {
this.fontStyle = fontStyle;
@ -393,11 +401,10 @@ export class ThemeTrieElement {
if (this._children.hasOwnProperty(head)) {
child = this._children[head];
} else {
child = new ThemeTrieElement(this._mainRule.clone());
child = new ThemeTrieElement(this._mainRule.clone(), ThemeTrieElementRule.cloneArr(this._rulesWithParentScopes));
this._children[head] = child;
}
// TODO: In the case that this element has `parentScopes`, should we generate one insert for each parentScope ?
child.insert(tail, parentScopes, fontStyle, foreground, background);
}

1
test-cases/themes/diff.css
Просмотреть файл

@ -10,6 +10,7 @@
white-space: pre;
}
.actual-line::before, .expected-line::before {
cursor: text;
content: '✓BOTH';
color: green;
width: 70px;

51
test-cases/themes/diff.js
Просмотреть файл

@ -1,6 +1,12 @@
var _allData = JSON.parse(atob(self.allData));
var output = document.createElement('div');
document.body.appendChild(output);
var escape = document.createElement('textarea');
function escapeHTML(str) {
str = str.replace(/\t/g, ' ');
escape.textContent = str;
return escape.innerHTML;
}
function renderTestCase(data) {
var content = data.testContent;
var output = [];
@ -16,7 +22,7 @@ function renderTestCase(data) {
while (actualLine.length > 0) {
var actualToken = actual[actualIndex++];
actualLine = actualLine.substr(actualToken.content.length);
actualLineOutput.push("<span style=\"color:" + actualToken.color.substring(0, 7) + "\" data-actual-index=\"" + (actualIndex - 1) + "\">" + actualToken.content + "</span>");
actualLineOutput.push("<span style=\"color:" + actualToken.color.substring(0, 7) + "\" data-actual-index=\"" + (actualIndex - 1) + "\">" + escapeHTML(actualToken.content) + "</span>");
}
actualLineOutput.push('</div>');
var expectedLineOutput = [];
@ -24,15 +30,15 @@ function renderTestCase(data) {
while (expectedLine.length > 0) {
var expectedToken = expected[expectedIndex++];
expectedLine = expectedLine.substr(expectedToken.content.length);
expectedLineOutput.push("<span style=\"color:" + expectedToken.color + "\" data-expected-index=\"" + (expectedIndex - 1) + "\">" + expectedToken.content + "</span>");
expectedLineOutput.push("<span style=\"color:" + expectedToken.color + "\" data-expected-index=\"" + (expectedIndex - 1) + "\">" + escapeHTML(expectedToken.content) + "</span>");
}
expectedLineOutput.push('</div>');
var diffOutput = [];
while (diffIndex < diff.length && diff[diffIndex].oldIndex < expectedIndex) {
diffOutput.push('<tr><td style="width:500px">');
diffOutput.push(JSON.stringify(diff[diffIndex].newToken, null, ' '));
diffOutput.push(escapeHTML(JSON.stringify(diff[diffIndex].newToken, null, ' ')));
diffOutput.push('</td><td style="width:500px">');
diffOutput.push(JSON.stringify(diff[diffIndex].oldToken, null, ' '));
diffOutput.push(escapeHTML(JSON.stringify(diff[diffIndex].oldToken, null, ' ')));
diffOutput.push('</td></tr>');
diffIndex++;
}
@ -102,17 +108,13 @@ document.body.onclick = function (e) {
targetDiff.className = 'diff collapsed';
}
};
var acceptBtn = document.createElement('button');
acceptBtn.innerHTML = 'Accept diff';
acceptBtn.style.position = 'fixed';
acceptBtn.style.top = '10px';
acceptBtn.style.right = '10px';
acceptBtn.style.fontSize = '150%';
document.body.appendChild(acceptBtn);
acceptBtn.onclick = function () {
var acceptDiffContent = (function () {
var result = {};
for (var i = 0, len = _allData.length; i < len; i++) {
var data = _allData[i];
if (!data.actual) {
continue;
}
result[data.themeName] = data.diff.map(function (diffEntry) {
return {
index: diffEntry.oldIndex,
@ -122,7 +124,24 @@ acceptBtn.onclick = function () {
};
});
}
console.log(JSON.stringify(result, null, '\t'));
return JSON.stringify(result, null, '\t');
})();
var diffTA = document.createElement('textarea');
diffTA.value = acceptDiffContent;
document.body.appendChild(diffTA);
document.body.oncopy = function (e) {
e.clipboardData.setData('text', acceptDiffContent);
};
var acceptBtn = document.createElement('button');
acceptBtn.innerHTML = 'Accept diff';
acceptBtn.style.position = 'fixed';
acceptBtn.style.top = '10px';
acceptBtn.style.right = '10px';
acceptBtn.style.fontSize = '150%';
document.body.appendChild(acceptBtn);
acceptBtn.onclick = function () {
diffTA.select();
console.log(acceptDiffContent);
};
var patchedBtn = document.createElement('button');
patchedBtn.innerHTML = 'View patched diff';
@ -143,6 +162,9 @@ originalBtn.onclick = renderOriginalDiff;
function renderOriginalDiff() {
output.innerHTML = '';
_allData.forEach(function (data) {
if (!data.actual) {
return;
}
output.appendChild(renderTestCase({
testContent: data.testContent,
themeName: data.themeName,
@ -156,6 +178,9 @@ function renderOriginalDiff() {
function renderPatchedDiff() {
output.innerHTML = '';
_allData.forEach(function (data) {
if (!data.actual) {
return;
}
output.appendChild(renderTestCase({
testContent: data.testContent,
themeName: data.themeName,

53
test-cases/themes/diff.ts
Просмотреть файл

@ -61,6 +61,13 @@ interface ITestCaseData {
diff: ITokenizationDiff[];
}
var escape = document.createElement('textarea');
function escapeHTML(str:string): string {
str = str.replace(/\t/g, ' ');
escape.textContent = str;
return escape.innerHTML;
}
function renderTestCase(data: ITestCaseData): HTMLElement {
let content = data.testContent;
@ -79,7 +86,7 @@ function renderTestCase(data: ITestCaseData): HTMLElement {
while (actualLine.length > 0) {
let actualToken = actual[actualIndex++];
actualLine = actualLine.substr(actualToken.content.length);
actualLineOutput.push(`<span style="color:${actualToken.color.substring(0, 7)}" data-actual-index="${actualIndex - 1}">${actualToken.content}</span>`);
actualLineOutput.push(`<span style="color:${actualToken.color.substring(0, 7)}" data-actual-index="${actualIndex - 1}">${escapeHTML(actualToken.content)}</span>`);
}
actualLineOutput.push('</div>');
@ -88,16 +95,16 @@ function renderTestCase(data: ITestCaseData): HTMLElement {
while (expectedLine.length > 0) {
let expectedToken = expected[expectedIndex++];
expectedLine = expectedLine.substr(expectedToken.content.length);
expectedLineOutput.push(`<span style="color:${expectedToken.color}" data-expected-index="${expectedIndex - 1}">${expectedToken.content}</span>`);
expectedLineOutput.push(`<span style="color:${expectedToken.color}" data-expected-index="${expectedIndex - 1}">${escapeHTML(expectedToken.content)}</span>`);
}
expectedLineOutput.push('</div>');
let diffOutput: string[] = [];
while (diffIndex < diff.length && diff[diffIndex].oldIndex < expectedIndex) {
diffOutput.push('<tr><td style="width:500px">');
diffOutput.push(JSON.stringify(diff[diffIndex].newToken, null, ' '));
diffOutput.push(escapeHTML(JSON.stringify(diff[diffIndex].newToken, null, ' ')));
diffOutput.push('</td><td style="width:500px">');
diffOutput.push(JSON.stringify(diff[diffIndex].oldToken, null, ' '));
diffOutput.push(escapeHTML(JSON.stringify(diff[diffIndex].oldToken, null, ' ')));
diffOutput.push('</td></tr>');
diffIndex++;
}
@ -170,18 +177,15 @@ document.body.onclick = function (e) {
}
};
let acceptBtn = document.createElement('button');
acceptBtn.innerHTML = 'Accept diff';
acceptBtn.style.position = 'fixed';
acceptBtn.style.top = '10px';
acceptBtn.style.right = '10px';
acceptBtn.style.fontSize = '150%';
document.body.appendChild(acceptBtn);
acceptBtn.onclick = function () {
let acceptDiffContent = (function() {
let result = {};
for (let i = 0, len = _allData.length; i < len; i++) {
let data = _allData[i];
if (!data.actual) {
continue;
}
result[data.themeName] = data.diff.map(function (diffEntry) {
return {
index: diffEntry.oldIndex,
@ -191,8 +195,25 @@ acceptBtn.onclick = function () {
};
});
}
return JSON.stringify(result, null, '\t');
})();
let diffTA = document.createElement('textarea');
diffTA.value = acceptDiffContent;
document.body.appendChild(diffTA);
document.body.oncopy = function(e) {
e.clipboardData.setData('text', acceptDiffContent);
};
console.log(JSON.stringify(result, null, '\t'));
let acceptBtn = document.createElement('button');
acceptBtn.innerHTML = 'Accept diff';
acceptBtn.style.position = 'fixed';
acceptBtn.style.top = '10px';
acceptBtn.style.right = '10px';
acceptBtn.style.fontSize = '150%';
document.body.appendChild(acceptBtn);
acceptBtn.onclick = function () {
diffTA.select();
console.log(acceptDiffContent);
};
let patchedBtn = document.createElement('button');
@ -216,6 +237,9 @@ originalBtn.onclick = renderOriginalDiff;
function renderOriginalDiff() {
output.innerHTML = '';
_allData.forEach(function (data) {
if (!data.actual) {
return;
}
output.appendChild(renderTestCase({
testContent: data.testContent,
themeName: data.themeName,
@ -230,6 +254,9 @@ function renderOriginalDiff() {
function renderPatchedDiff() {
output.innerHTML = '';
_allData.forEach(function (data) {
if (!data.actual) {
return;
}
output.appendChild(renderTestCase({
testContent: data.testContent,
themeName: data.themeName,