Add infrastructure for us test our own Razor VSCode grammar.

- In order to programatically parse anything with the Razor grammar we need to reconstruct an environment that's similar to VSCode where grammar's for C#, HTML, JavaScript and CSS exist. To do this I grabbed all of Razor's embedded language grammars to ensure we can construct a valid Razor TextMate grammar parser.
- Copied existing unit test boiler plate (jest.config.js etc.) to a new `Microsoft.AspNetCore.Razor.VSCode.Grammar.Test` project.
- Added VSCode utilities to enable running and debugging of grammar tests directly in Visual Studio code.
- Built test utilities to:
  1. Tokenize a content with Razor's grammar.
  2. Generate snapshot contents (a serialized form of tokenized content).
- Used Jest's [built-in snapshot testing](https://jestjs.io/docs/en/snapshot-testing) to build a simple testing suite that Tokenizes Razor content -> Serializes it -> Compares it to a baseline (or updates).
- Ensured that command line testing works as expected as well via `yarn jest` to run tests and `yarn jest -u` to update snapshots for tests.
- Added an escaped transitions grammar test as an example to show how all future tests will be constructed.

aspnet/AspNetCore#14287

.vscode/launch.json

@@ -40,6 +40,20 @@
             "internalConsoleOptions": "openOnSessionStart",
             "preLaunchTask": "CompileUnitTests"
         },
+        {
+            "type": "node",
+            "request": "launch",
+            "name": "Run Grammar Tests",
+            "runtimeExecutable": "yarn",
+            "cwd": "${workspaceFolder}/src/Razor/test/Microsoft.AspNetCore.Razor.VSCode.Grammar.Test",
+            "runtimeArgs": [
+                "test:debug"
+            ],
+            "port": 9229,
+            "sourceMaps": true,
+            "internalConsoleOptions": "openOnSessionStart",
+            "preLaunchTask": "CompileGrammarTests"
+        },
         {
             "name": "Run Functional Tests",
             "type": "extensionHost",

.vscode/tasks.json

@@ -85,6 +85,20 @@
                 "reveal": "silent"
             }
         },
+        {
+            "label": "CompileGrammarTests",
+            "command": "dotnet",
+            "args": [
+                "build"
+            ],
+            "options": {
+                "cwd": "src/Razor/test/Microsoft.AspNetCore.Razor.VSCode.Grammar.Test/"
+            },
+            "problemMatcher": "$tsc-watch",
+            "presentation": {
+                "reveal": "silent"
+            }
+        },
         {
             "label": "CompileFunctionalTest",
             "command": "dotnet",

src/Razor/test/Microsoft.AspNetCore.Razor.VSCode.Grammar.Test/Microsoft.AspNetCore.Razor.VSCode.Grammar.Test.npmproj

@@ -0,0 +1,27 @@
+<Project DefaultTargets="Build">
+
+  <Import Project="$([MSBuild]::GetDirectoryNameOfFileAbove($(MSBuildThisFileDirectory), Directory.Build.props))\Directory.Build.props" />
+
+  <PropertyGroup>
+    <TargetFramework>netstandard2.0</TargetFramework>
+    <IsTestProject>true</IsTestProject>
+    <IsPackable>false</IsPackable>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <ProjectReference
+      Include="..\..\src\Microsoft.AspNetCore.Razor.VSCode.Extension\Microsoft.AspNetCore.Razor.VSCode.Extension.npmproj"
+      ReferenceOutputAssemblies="false"
+      SkipGetTargetFrameworkProperties="true"
+      UndefineProperties="TargetFramework"
+      Private="false" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <BuildOutputFiles Include="dist\infrastructure\SnapshotTests.js" />
+  </ItemGroup>
+
+
+  <Import Project="$([MSBuild]::GetDirectoryNameOfFileAbove($(MSBuildThisFileDirectory), Directory.Build.targets))\Directory.Build.targets" />
+
+</Project>

src/Razor/test/Microsoft.AspNetCore.Razor.VSCode.Grammar.Test/jest.config.js

@@ -0,0 +1,15 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+module.exports = {
+  globals: {
+    "ts-jest": {
+      "tsConfig": "./tsconfig.json",
+      "babeConfig": true,
+      "diagnostics": true
+    }
+  },
+  testPathIgnorePatterns: [ 'dist' ],
+  preset: 'ts-jest',
+  testEnvironment: 'jsdom'
+};

src/Razor/test/Microsoft.AspNetCore.Razor.VSCode.Grammar.Test/package.json

@@ -0,0 +1,24 @@
+{
+  "name": "razor-vscode-grammar-test",
+  "private": true,
+  "displayName": "Razor Grammar Tests",
+  "scripts": {
+    "clean": "rimraf dist",
+    "build": "yarn run clean && yarn run lint && tsc -p ./",
+    "lint": "tslint --project ./",
+    "test": "jest",
+    "test:debug": "node --inspect-brk ./node_modules/jest/bin/jest.js --runInBand --colors"
+  },
+  "devDependencies": {
+    "@types/jest": "^24.0.6",
+    "@types/node": "9.4.7",
+    "jest": "^24.8.0",
+    "ts-jest": "^24.0.0",
+    "ts-node": "^7.0.1",
+    "tslint": "^5.11.0",
+    "typescript": "3.3.4000",
+    "rimraf": "2.6.3",
+    "vscode-textmate": "4.4.0",
+    "oniguruma": "7.2.1"
+  }
+}

src/Razor/test/Microsoft.AspNetCore.Razor.VSCode.Grammar.Test/tests/GrammarTests.test.ts

@@ -0,0 +1,15 @@
+/* --------------------------------------------------------------------------------------------
+* Copyright (c) Microsoft Corporation. All rights reserved.
+* Licensed under the MIT License. See License.txt in the project root for license information.
+* ------------------------------------------------------------------------------------------ */
+
+import { RunTransitionsSuite } from './Transitions';
+
+// We bring together all test suites and wrap them in one here. The reason behind this is that
+// modules get reloaded per test suite and the vscode-textmate library doesn't support the way
+// that Jest reloads those modules. By wrapping all suites in one we can guaruntee that the
+// modules don't get torn down inbetween suites.
+
+describe('Grammar tests', () => {
+    RunTransitionsSuite();
+});

src/Razor/test/Microsoft.AspNetCore.Razor.VSCode.Grammar.Test/tests/Transitions.ts

@@ -0,0 +1,16 @@
+/* --------------------------------------------------------------------------------------------
+ * Copyright (c) Microsoft Corporation. All rights reserved.
+ * Licensed under the MIT License. See License.txt in the project root for license information.
+ * ------------------------------------------------------------------------------------------ */
+
+import { assertMatchesSnapshot } from './infrastructure/TestUtilities';
+
+// See GrammarTests.test.ts for details on exporting this test suite instead of running in place.
+
+export function RunTransitionsSuite() {
+    describe('Transitions', () => {
+        it('Escaped transitions', async () => {
+            await assertMatchesSnapshot('@@');
+        });
+    });
+}

src/Razor/test/Microsoft.AspNetCore.Razor.VSCode.Grammar.Test/tests/__snapshots__/GrammarTests.test.ts.snap

@@ -0,0 +1,7 @@
+// Jest Snapshot v1, https://goo.gl/fbAQLP
+
+exports[`Grammar tests Transitions Escaped transitions 1`] = `
+"Line: @@
+ - token from 0 to 2 (@@) with scopes text.aspnetcorerazor, constant.character.escape.razor.transition
+"
+`;

src/Razor/test/Microsoft.AspNetCore.Razor.VSCode.Grammar.Test/tests/infrastructure/ITokenizedContent.ts

@@ -0,0 +1,12 @@
+/* --------------------------------------------------------------------------------------------
+* Copyright (c) Microsoft Corporation. All rights reserved.
+* Licensed under the MIT License. See License.txt in the project root for license information.
+* ------------------------------------------------------------------------------------------ */
+
+import { ITokenizeLineResult } from 'vscode-textmate';
+
+export interface ITokenizedContent {
+    readonly source: string;
+    readonly lines: string[];
+    readonly tokenizedLines: ITokenizeLineResult[];
+}

src/Razor/test/Microsoft.AspNetCore.Razor.VSCode.Grammar.Test/tests/infrastructure/SnapshotFactory.ts

@@ -0,0 +1,25 @@
+/* --------------------------------------------------------------------------------------------
+ * Copyright (c) Microsoft Corporation. All rights reserved.
+ * Licensed under the MIT License. See License.txt in the project root for license information.
+ * ------------------------------------------------------------------------------------------ */
+
+import { ITokenizedContent } from './ITokenizedContent';
+
+export function createSnapshot(tokenizedContent: ITokenizedContent): string {
+    const snapshotLines: string[] = [];
+    for (let i = 0; i < tokenizedContent.tokenizedLines.length; i++) {
+        const line = tokenizedContent.lines[i];
+        const tokenizedLine = tokenizedContent.tokenizedLines[i];
+
+        snapshotLines.push(`Line: ${line}`);
+        for (const token of tokenizedLine.tokens) {
+            snapshotLines.push(` - token from ${token.startIndex} to ${token.endIndex} ` +
+                `(${line.substring(token.startIndex, token.endIndex)}) ` +
+                `with scopes ${token.scopes.join(', ')}`);
+        }
+        snapshotLines.push('');
+    }
+
+    const snapshot = snapshotLines.join('\n');
+    return snapshot;
+}

src/Razor/test/Microsoft.AspNetCore.Razor.VSCode.Grammar.Test/tests/infrastructure/TestUtilities.ts

@@ -0,0 +1,14 @@
+
+/* --------------------------------------------------------------------------------------------
+* Copyright (c) Microsoft Corporation. All rights reserved.
+* Licensed under the MIT License. See License.txt in the project root for license information.
+* ------------------------------------------------------------------------------------------ */
+
+import { createSnapshot } from './SnapshotFactory';
+import { tokenize } from './TokenizedContentProvider';
+
+export async function assertMatchesSnapshot(content: string) {
+    const tokenizedContent = await tokenize(content);
+    const currentSnapshot = createSnapshot(tokenizedContent);
+    expect(currentSnapshot).toMatchSnapshot();
+}

src/Razor/test/Microsoft.AspNetCore.Razor.VSCode.Grammar.Test/tests/infrastructure/TokenizedContentProvider.ts

@@ -0,0 +1,88 @@
+/* --------------------------------------------------------------------------------------------
+ * Copyright (c) Microsoft Corporation. All rights reserved.
+ * Licensed under the MIT License. See License.txt in the project root for license information.
+ * ------------------------------------------------------------------------------------------ */
+
+import * as fs from 'fs';
+import { IGrammar, INITIAL, IRawGrammar, ITokenizeLineResult, parseRawGrammar, Registry } from 'vscode-textmate';
+import { ITokenizedContent } from './ITokenizedContent';
+
+let razorGrammarCache: IGrammar | undefined;
+
+export async function tokenize(source: string) {
+    const lines = source.split('\n');
+    const grammar = await loadRazorGrammar();
+    const tokenizedLines: ITokenizeLineResult[] = [];
+
+    let ruleStack = INITIAL;
+    for (const line of lines) {
+        const tokenizedLine = grammar.tokenizeLine(line, ruleStack);
+        tokenizedLines.push(tokenizedLine);
+        ruleStack = tokenizedLine.ruleStack;
+    }
+
+    const tokenizedContent: ITokenizedContent = {
+        source,
+        lines,
+        tokenizedLines,
+    };
+    return tokenizedContent;
+}
+
+async function loadRazorGrammar() {
+    if (!razorGrammarCache) {
+        const registry = new Registry({
+            loadGrammar: loadRawGrammarFromScope,
+        });
+
+        const razorGrammar = await registry.loadGrammar('text.aspnetcorerazor');
+        if (!razorGrammar) {
+            throw new Error('Could not load Razor grammar');
+        }
+
+        razorGrammarCache = razorGrammar;
+    }
+
+    return razorGrammarCache;
+}
+
+async function loadRawGrammarFromScope(scopeName: string) {
+    const scopeToRawGrammarFilePath = await getScopeToFilePathRegistry();
+    const grammar = scopeToRawGrammarFilePath[scopeName];
+    if (!grammar) {
+        // Unknown scope
+        throw new Error(`Unknown scope name when loading raw grammar: ${scopeName}`);
+    }
+
+    return grammar;
+}
+
+async function loadRawGrammar(filePath: string) {
+    const fileBuffer = await readFile(filePath);
+    const fileContent = fileBuffer.toString();
+    const rawGrammar = parseRawGrammar(fileContent, filePath);
+    return rawGrammar;
+}
+
+async function getScopeToFilePathRegistry() {
+    const razorRawGrammar = await loadRawGrammar('../../src/Microsoft.AspNetCore.Razor.VSCode.Extension/syntaxes/aspnetcorerazor.tmLanguage.json');
+    const htmlRawGrammar = await loadRawGrammar('embeddedGrammars/html.tmLanguage.json');
+    const cssRawGrammar = await loadRawGrammar('embeddedGrammars/css.tmLanguage.json');
+    const javaScriptRawGrammar = await loadRawGrammar('embeddedGrammars/JavaScript.tmLanguage.json');
+    const csharpRawGrammar = await loadRawGrammar('embeddedGrammars/csharp.tmLanguage.json');
+    const scopeToRawGrammarFilePath: { [key: string]: IRawGrammar } = {
+        'text.aspnetcorerazor': razorRawGrammar,
+        'text.html.basic': htmlRawGrammar,
+        'source.css': cssRawGrammar,
+        'source.js': javaScriptRawGrammar,
+        'source.cs': csharpRawGrammar,
+    };
+
+    return scopeToRawGrammarFilePath;
+}
+
+function readFile(filePath: string): Promise<Buffer> {
+    return new Promise((resolve, reject) => {
+        fs.readFile(filePath, (error, data) => error ? reject(error) : resolve(data));
+    });
+}

src/Razor/test/Microsoft.AspNetCore.Razor.VSCode.Grammar.Test/tsconfig.json

@@ -0,0 +1,9 @@
+{
+	"extends": "../../tsconfig.json",
+	"compilerOptions": {
+		"outDir": "dist"
+	},
+	"include": [
+	  "tests/**/*"
+	]
+  }

src/Razor/test/Microsoft.AspNetCore.Razor.VSCode.Grammar.Test/tslint.json

@@ -0,0 +1,3 @@
+{
+    "extends": "../../tslint.json"
+}