diff --git a/src/tests/themeTest.ts b/src/tests/themeTest.ts index 7447ff8..9aec52b 100644 --- a/src/tests/themeTest.ts +++ b/src/tests/themeTest.ts @@ -6,7 +6,7 @@ import * as fs from 'fs'; import * as path from 'path'; import { IEmbeddedLanguagesMap } from '../main'; -import { tokenizeWithTheme, IThemedToken } from './themedTokenizer'; +import { tokenizeWithTheme, IThemedToken, IThemedTokenScopeExplanation } from './themedTokenizer'; import { Resolver, ThemeData } from './themes.test'; interface IExpected { @@ -69,7 +69,6 @@ export class ThemeTest { const testFileExpected = ThemeTest._readJSONFile(EXPECTED_FILE_PATH); const EXPECTED_PATCH_FILE_PATH = path.join(THEMES_TEST_PATH, 'tests', testFile + '.result.patch'); - console.log(EXPECTED_PATCH_FILE_PATH); const testFileExpectedPatch = ThemeTest._readJSONFile(EXPECTED_PATCH_FILE_PATH); // Determine the language @@ -142,7 +141,8 @@ export class ThemeTest { public hasDiff(): boolean { for (let i = 0; i < this.tests.length; i++) { - if (this.tests[i].patchedDiff.length > 0) { + let test = this.tests[i]; + if (test.patchedDiff && test.patchedDiff.length > 0) { return true; } } @@ -162,10 +162,22 @@ export class ThemeTest { } } +interface IActualCanonicalToken { + content: string; + color: string; + scopes: IThemedTokenScopeExplanation[]; +} +interface IExpectedCanonicalToken { + oldIndex: number; + content: string; + color: string; + _r: string; + _t: string; +} interface ITokenizationDiff { oldIndex: number; oldToken: IExpectedTokenization; - newToken: IThemedToken; + newToken: IActualCanonicalToken; } interface IDiffPageData { @@ -215,16 +227,46 @@ class SingleThemeTest { this.expected = expected; this.expectedPatch = expectedPatch; - this.patchedExpected = this.expected.slice(0); - for (let i = 0; i < this.expectedPatch.length; i++) { - let patch = this.expectedPatch[i]; + this.patchedExpected = []; + let patchIndex = this.expectedPatch.length - 1; + for (let i = this.expected.length - 1; i >= 0; i--) { + let expectedElement = this.expected[i]; + let content = expectedElement.content; + while (patchIndex >= 0 && i === this.expectedPatch[patchIndex].index) { + let patch = this.expectedPatch[patchIndex]; - this.patchedExpected[patch.index] = { - _r: this.patchedExpected[patch.index]._r, - _t: this.patchedExpected[patch.index]._t, - content: patch.content, - color: patch.newColor - }; + let patchContentIndex = content.lastIndexOf(patch.content); + + let afterContent = content.substr(patchContentIndex + patch.content.length); + if (afterContent.length > 0) { + this.patchedExpected.unshift({ + _r: expectedElement._r, + _t: expectedElement._t, + content: afterContent, + color: expectedElement.color + }); + } + + this.patchedExpected.unshift({ + _r: expectedElement._r, + _t: expectedElement._t, + content: patch.content, + color: patch.newColor + }); + + content = content.substr(0, patchContentIndex); + + patchIndex--; + } + + if (content.length > 0) { + this.patchedExpected.unshift({ + _r: expectedElement._r, + _t: expectedElement._t, + content: content, + color: expectedElement.color + }); + } } this.backgroundColor = null; @@ -272,67 +314,69 @@ class SingleThemeTest { }); } - private static computeThemeTokenizationDiff(actual: IThemedToken[], expected: IExpectedTokenization[]): ITokenizationDiff[] { + private static computeThemeTokenizationDiff(_actual: IThemedToken[], _expected: IExpectedTokenization[]): ITokenizationDiff[] { + let canonicalTokens: string[] = []; + for (let i = 0, len = _actual.length; i < len; i++) { + let explanation = _actual[i].explanation; + for (let j = 0, lenJ = explanation.length; j < lenJ; j++) { + canonicalTokens.push(explanation[j].content); + } + } + + let actual: IActualCanonicalToken[] = []; + for (let i = 0, len = _actual.length; i < len; i++) { + let item = _actual[i]; + + for (let j = 0, lenJ = item.explanation.length; j < lenJ; j++) { + actual.push({ + content: item.explanation[j].content, + color: item.color, + scopes: item.explanation[j].scopes + }); + } + } + + let expected: IExpectedCanonicalToken[] = []; + for (let i = 0, len = _expected.length, canonicalIndex = 0; i < len; i++) { + let item = _expected[i]; + + let content = item.content; + while (content.length > 0) { + expected.push({ + oldIndex: i, + content: canonicalTokens[canonicalIndex], + color: item.color, + _t: item._t, + _r: item._r + }); + content = content.substr(canonicalTokens[canonicalIndex].length); + canonicalIndex++; + } + } + + if (actual.length !== expected.length) { + throw new Error('Content mismatch'); + } + let diffs: ITokenizationDiff[] = []; - let i = 0, j = 0, len = actual.length, lenJ = expected.length; - do { - if (i >= len && j >= lenJ) { - // ok - break; + for (let i = 0, len = actual.length; i < len; i++) { + let expectedItem = expected[i]; + let actualItem = actual[i]; + + let contentIsInvisible = /^\s+$/.test(expectedItem.content); + if (contentIsInvisible) { + continue; } - if (i >= len) { - // will fail - throw new Error('Reached end of actual before end of expected'); + if (actualItem.color.substr(0, 7) !== expectedItem.color) { + diffs.push({ + oldIndex: expectedItem.oldIndex, + oldToken: expectedItem, + newToken: actualItem + }); } - - if (j >= lenJ) { - // will fail - throw new Error('Reached end of expected before end of actual'); - } - - let actualContent = actual[i].content; - let actualColor = actual[i].color; - if (actualColor.length > 7) { - // TODO: remove alpha to match expected tests format - actualColor = actualColor.substring(0, 7); - } - - while (actualContent.length > 0 && j < lenJ) { - let expectedContent = expected[j].content; - let expectedColor = expected[j].color; - - let contentIsInvisible = /^\s+$/.test(expectedContent); - if (!contentIsInvisible && actualColor !== expectedColor) { - // console.log('COLOR MISMATCH: ', actualColor, expectedColor); - // select the same token from the explanation - let reducedExplanation = actual[i].explanation.filter((e) => e.content === expectedContent); - if (reducedExplanation.length === 0) { - reducedExplanation = actual[i].explanation; - } - diffs.push({ - oldIndex: j, - oldToken: expected[j], - newToken: { - content: actual[i].content, - color: actual[i].color, - explanation: reducedExplanation - } - }); - } - - if (actualContent.substr(0, expectedContent.length) !== expectedContent) { - throw new Error(`at ${actualContent} (${i}-${j}), content mismatch: ${actualContent}, ${expectedContent}`); - } - - actualContent = actualContent.substr(expectedContent.length); - - j++; - } - - i++; - } while (true); + } return diffs; } diff --git a/src/tests/themes.test.ts b/src/tests/themes.test.ts index f2487eb..55d85fe 100644 --- a/src/tests/themes.test.ts +++ b/src/tests/themes.test.ts @@ -7,6 +7,7 @@ import * as fs from 'fs'; import * as path from 'path'; import * as assert from 'assert'; import { Registry, RegistryOptions, IRawTheme } from '../main'; +import { ScopeListElement, ScopeMetadata, StackElementMetadata } from '../grammar'; import { Theme, strcmp, strArrCmp, ThemeTrieElement, ThemeTrieElementRule, parseTheme, ParsedThemeRule, FontStyle, ColorMap @@ -111,7 +112,7 @@ export class Resolver implements RegistryOptions { return path.join(THEMES_TEST_PATH, grammar.path); } } - console.warn('missing gramamr for ' + scopeName); + // console.warn('missing grammar for ' + scopeName); } } @@ -165,12 +166,8 @@ class ThemeInfo { function assertThemeTest(test: ThemeTest, themeDatas: ThemeData[]): void { it(test.testName, (done) => { test.evaluate(themeDatas, (err) => { - console.log('HERE I AM!!'); - test.writeDiffPage(); - assert.ok(!test.hasDiff(), 'no more unpatched differences'); - done(); }); }).timeout(5000); @@ -217,6 +214,49 @@ function assertThemeTest(test: ThemeTest, themeDatas: ThemeData[]): void { })(); describe('Theme matching', () => { + it('gives higher priority to parent matches 1', () => { + let theme = Theme.createFromRawTheme({ + settings: [ + { settings: { foreground: '#100000', background: '#200000' } }, + { scope: 'c a', settings: { foreground: '#300000' } }, + { scope: 'd a.b', settings: { foreground: '#400000' } }, + { scope: 'a', settings: { foreground: '#500000' } }, + ] + }); + + let colorMap = new ColorMap(); + const _NOT_SET = 0; + const _A = colorMap.getId('#100000'); + const _B = colorMap.getId('#200000'); + const _C = colorMap.getId('#500000'); + const _D = colorMap.getId('#300000'); + const _E = colorMap.getId('#400000'); + + let actual = theme.match('a.b'); + + assert.deepEqual(actual, [ + new ThemeTrieElementRule(['d'], FontStyle.NotSet, _E, _NOT_SET), + new ThemeTrieElementRule(['c'], FontStyle.NotSet, _D, _NOT_SET), + new ThemeTrieElementRule(null, FontStyle.NotSet, _C, _NOT_SET), + ]); + }); + + it('gives higher priority to parent matches 3', () => { + let theme = Theme.createFromRawTheme({ + settings: [ + { settings: { foreground: '#100000', background: '#200000' } }, + { scope: 'meta.tag entity', settings: { foreground: '#300000' } }, + { scope: 'meta.selector.css entity.name.tag', settings: { foreground: '#400000' } }, + { scope: 'entity', settings: { foreground: '#500000' } }, + ] + }); + + let root = new ScopeListElement(null, 'text.html.cshtml', 0); + let parent = new ScopeListElement(root, 'meta.tag.structure.any.html', 0); + let r = ScopeListElement.mergeMetadata(0, parent, new ScopeMetadata('entity.name.tag.structure.any.html', 0, 0, theme.match('entity.name.tag.structure.any.html'))); + let colorMap = theme.getColorMap(); + assert.equal(colorMap[StackElementMetadata.getForeground(r)], '#300000'); + }); it('can match', () => { let theme = Theme.createFromRawTheme({ settings: [ diff --git a/src/theme.ts b/src/theme.ts index abf73b4..3f73cd8 100644 --- a/src/theme.ts +++ b/src/theme.ts @@ -299,6 +299,14 @@ export class ThemeTrieElementRule { return new ThemeTrieElementRule(this.parentScopes, this.fontStyle, this.foreground, this.background); } + public static cloneArr(arr:ThemeTrieElementRule[]): ThemeTrieElementRule[] { + let r: ThemeTrieElementRule[] = []; + for (let i = 0, len = arr.length; i < len; i++) { + r[i] = arr[i].clone(); + } + return r; + } + public acceptOverwrite(fontStyle: number, foreground: number, background: number): void { if (fontStyle !== FontStyle.NotSet) { this.fontStyle = fontStyle; @@ -393,11 +401,10 @@ export class ThemeTrieElement { if (this._children.hasOwnProperty(head)) { child = this._children[head]; } else { - child = new ThemeTrieElement(this._mainRule.clone()); + child = new ThemeTrieElement(this._mainRule.clone(), ThemeTrieElementRule.cloneArr(this._rulesWithParentScopes)); this._children[head] = child; } - // TODO: In the case that this element has `parentScopes`, should we generate one insert for each parentScope ? child.insert(tail, parentScopes, fontStyle, foreground, background); } diff --git a/test-cases/themes/diff.css b/test-cases/themes/diff.css index 73ccc0f..dd56485 100644 --- a/test-cases/themes/diff.css +++ b/test-cases/themes/diff.css @@ -10,6 +10,7 @@ white-space: pre; } .actual-line::before, .expected-line::before { + cursor: text; content: '✓BOTH'; color: green; width: 70px; diff --git a/test-cases/themes/diff.js b/test-cases/themes/diff.js index 91bf17c..23a4988 100644 --- a/test-cases/themes/diff.js +++ b/test-cases/themes/diff.js @@ -1,6 +1,12 @@ var _allData = JSON.parse(atob(self.allData)); var output = document.createElement('div'); document.body.appendChild(output); +var escape = document.createElement('textarea'); +function escapeHTML(str) { + str = str.replace(/\t/g, ' '); + escape.textContent = str; + return escape.innerHTML; +} function renderTestCase(data) { var content = data.testContent; var output = []; @@ -16,7 +22,7 @@ function renderTestCase(data) { while (actualLine.length > 0) { var actualToken = actual[actualIndex++]; actualLine = actualLine.substr(actualToken.content.length); - actualLineOutput.push("" + actualToken.content + ""); + actualLineOutput.push("" + escapeHTML(actualToken.content) + ""); } actualLineOutput.push(''); var expectedLineOutput = []; @@ -24,15 +30,15 @@ function renderTestCase(data) { while (expectedLine.length > 0) { var expectedToken = expected[expectedIndex++]; expectedLine = expectedLine.substr(expectedToken.content.length); - expectedLineOutput.push("" + expectedToken.content + ""); + expectedLineOutput.push("" + escapeHTML(expectedToken.content) + ""); } expectedLineOutput.push(''); var diffOutput = []; while (diffIndex < diff.length && diff[diffIndex].oldIndex < expectedIndex) { diffOutput.push(''); - diffOutput.push(JSON.stringify(diff[diffIndex].newToken, null, ' ')); + diffOutput.push(escapeHTML(JSON.stringify(diff[diffIndex].newToken, null, ' '))); diffOutput.push(''); - diffOutput.push(JSON.stringify(diff[diffIndex].oldToken, null, ' ')); + diffOutput.push(escapeHTML(JSON.stringify(diff[diffIndex].oldToken, null, ' '))); diffOutput.push(''); diffIndex++; } @@ -102,17 +108,13 @@ document.body.onclick = function (e) { targetDiff.className = 'diff collapsed'; } }; -var acceptBtn = document.createElement('button'); -acceptBtn.innerHTML = 'Accept diff'; -acceptBtn.style.position = 'fixed'; -acceptBtn.style.top = '10px'; -acceptBtn.style.right = '10px'; -acceptBtn.style.fontSize = '150%'; -document.body.appendChild(acceptBtn); -acceptBtn.onclick = function () { +var acceptDiffContent = (function () { var result = {}; for (var i = 0, len = _allData.length; i < len; i++) { var data = _allData[i]; + if (!data.actual) { + continue; + } result[data.themeName] = data.diff.map(function (diffEntry) { return { index: diffEntry.oldIndex, @@ -122,7 +124,24 @@ acceptBtn.onclick = function () { }; }); } - console.log(JSON.stringify(result, null, '\t')); + return JSON.stringify(result, null, '\t'); +})(); +var diffTA = document.createElement('textarea'); +diffTA.value = acceptDiffContent; +document.body.appendChild(diffTA); +document.body.oncopy = function (e) { + e.clipboardData.setData('text', acceptDiffContent); +}; +var acceptBtn = document.createElement('button'); +acceptBtn.innerHTML = 'Accept diff'; +acceptBtn.style.position = 'fixed'; +acceptBtn.style.top = '10px'; +acceptBtn.style.right = '10px'; +acceptBtn.style.fontSize = '150%'; +document.body.appendChild(acceptBtn); +acceptBtn.onclick = function () { + diffTA.select(); + console.log(acceptDiffContent); }; var patchedBtn = document.createElement('button'); patchedBtn.innerHTML = 'View patched diff'; @@ -143,6 +162,9 @@ originalBtn.onclick = renderOriginalDiff; function renderOriginalDiff() { output.innerHTML = ''; _allData.forEach(function (data) { + if (!data.actual) { + return; + } output.appendChild(renderTestCase({ testContent: data.testContent, themeName: data.themeName, @@ -156,6 +178,9 @@ function renderOriginalDiff() { function renderPatchedDiff() { output.innerHTML = ''; _allData.forEach(function (data) { + if (!data.actual) { + return; + } output.appendChild(renderTestCase({ testContent: data.testContent, themeName: data.themeName, diff --git a/test-cases/themes/diff.ts b/test-cases/themes/diff.ts index f6856b2..06d1458 100644 --- a/test-cases/themes/diff.ts +++ b/test-cases/themes/diff.ts @@ -61,6 +61,13 @@ interface ITestCaseData { diff: ITokenizationDiff[]; } +var escape = document.createElement('textarea'); +function escapeHTML(str:string): string { + str = str.replace(/\t/g, ' '); + escape.textContent = str; + return escape.innerHTML; +} + function renderTestCase(data: ITestCaseData): HTMLElement { let content = data.testContent; @@ -79,7 +86,7 @@ function renderTestCase(data: ITestCaseData): HTMLElement { while (actualLine.length > 0) { let actualToken = actual[actualIndex++]; actualLine = actualLine.substr(actualToken.content.length); - actualLineOutput.push(`${actualToken.content}`); + actualLineOutput.push(`${escapeHTML(actualToken.content)}`); } actualLineOutput.push(''); @@ -88,16 +95,16 @@ function renderTestCase(data: ITestCaseData): HTMLElement { while (expectedLine.length > 0) { let expectedToken = expected[expectedIndex++]; expectedLine = expectedLine.substr(expectedToken.content.length); - expectedLineOutput.push(`${expectedToken.content}`); + expectedLineOutput.push(`${escapeHTML(expectedToken.content)}`); } expectedLineOutput.push(''); let diffOutput: string[] = []; while (diffIndex < diff.length && diff[diffIndex].oldIndex < expectedIndex) { diffOutput.push(''); - diffOutput.push(JSON.stringify(diff[diffIndex].newToken, null, ' ')); + diffOutput.push(escapeHTML(JSON.stringify(diff[diffIndex].newToken, null, ' '))); diffOutput.push(''); - diffOutput.push(JSON.stringify(diff[diffIndex].oldToken, null, ' ')); + diffOutput.push(escapeHTML(JSON.stringify(diff[diffIndex].oldToken, null, ' '))); diffOutput.push(''); diffIndex++; } @@ -170,18 +177,15 @@ document.body.onclick = function (e) { } }; -let acceptBtn = document.createElement('button'); -acceptBtn.innerHTML = 'Accept diff'; -acceptBtn.style.position = 'fixed'; -acceptBtn.style.top = '10px'; -acceptBtn.style.right = '10px'; -acceptBtn.style.fontSize = '150%'; -document.body.appendChild(acceptBtn); -acceptBtn.onclick = function () { +let acceptDiffContent = (function() { let result = {}; for (let i = 0, len = _allData.length; i < len; i++) { let data = _allData[i]; + if (!data.actual) { + continue; + } + result[data.themeName] = data.diff.map(function (diffEntry) { return { index: diffEntry.oldIndex, @@ -191,8 +195,25 @@ acceptBtn.onclick = function () { }; }); } + return JSON.stringify(result, null, '\t'); +})(); +let diffTA = document.createElement('textarea'); +diffTA.value = acceptDiffContent; +document.body.appendChild(diffTA); +document.body.oncopy = function(e) { + e.clipboardData.setData('text', acceptDiffContent); +}; - console.log(JSON.stringify(result, null, '\t')); +let acceptBtn = document.createElement('button'); +acceptBtn.innerHTML = 'Accept diff'; +acceptBtn.style.position = 'fixed'; +acceptBtn.style.top = '10px'; +acceptBtn.style.right = '10px'; +acceptBtn.style.fontSize = '150%'; +document.body.appendChild(acceptBtn); +acceptBtn.onclick = function () { + diffTA.select(); + console.log(acceptDiffContent); }; let patchedBtn = document.createElement('button'); @@ -216,6 +237,9 @@ originalBtn.onclick = renderOriginalDiff; function renderOriginalDiff() { output.innerHTML = ''; _allData.forEach(function (data) { + if (!data.actual) { + return; + } output.appendChild(renderTestCase({ testContent: data.testContent, themeName: data.themeName, @@ -230,6 +254,9 @@ function renderOriginalDiff() { function renderPatchedDiff() { output.innerHTML = ''; _allData.forEach(function (data) { + if (!data.actual) { + return; + } output.appendChild(renderTestCase({ testContent: data.testContent, themeName: data.themeName,