diff --git a/extensions/ql-vscode/.eslintrc.js b/extensions/ql-vscode/.eslintrc.js index 503485444..a6a4d56b6 100644 --- a/extensions/ql-vscode/.eslintrc.js +++ b/extensions/ql-vscode/.eslintrc.js @@ -121,15 +121,6 @@ module.exports = { }, }, }, - { - files: ["test/**/*"], - parserOptions: { - project: resolve(__dirname, "test/tsconfig.json"), - }, - env: { - jest: true, - }, - }, { files: ["test/vscode-tests/**/*"], parserOptions: { @@ -156,6 +147,18 @@ module.exports = { ], }, }, + { + files: ["test/**/*"], + parserOptions: { + project: resolve(__dirname, "test/tsconfig.json"), + }, + env: { + jest: true, + }, + rules: { + "@typescript-eslint/no-explicit-any": "off", + }, + }, { files: [ ".eslintrc.js", @@ -188,11 +191,5 @@ module.exports = { "import/no-namespace": ["error", { ignore: ["react"] }], }, }, - { - files: ["test/**/*", "gulpfile.ts/**/*"], - rules: { - "@typescript-eslint/no-explicit-any": "off", - }, - }, ], }; diff --git a/extensions/ql-vscode/gulpfile.ts/textmate-grammar.ts b/extensions/ql-vscode/gulpfile.ts/textmate-grammar.ts new file mode 100644 index 000000000..ab61bec51 --- /dev/null +++ b/extensions/ql-vscode/gulpfile.ts/textmate-grammar.ts @@ -0,0 +1,91 @@ +/** + * A subset of the standard TextMate grammar that is used by our transformation + * step. For a full JSON schema, see: + * https://github.com/martinring/tmlanguage/blob/478ad124a21933cd4b0b65f1ee7ee18ee1f87473/tmlanguage.json + */ +export interface TextmateGrammar { + patterns: Pattern[]; + repository?: Record; +} + +/** + * The extended TextMate grammar as used by our transformation step. This is a superset of the + * standard TextMate grammar, and includes additional fields that are used by our transformation + * step. + * + * Any comment of the form `(?#ref-id)` in a `match`, `begin`, or `end` property will be replaced + * with the match text of the rule named "ref-id". If the rule named "ref-id" consists of just a + * `patterns` property with a list of `include` directives, the replacement pattern is the + * disjunction of the match patterns of all of the included rules. + */ +export interface ExtendedTextmateGrammar { + /** + * This represents the set of regular expression options to apply to all regular + * expressions throughout the file. + */ + regexOptions?: string; + /** + * This element defines a map of macro names to replacement text. When a `match`, `begin`, or + * `end` property has a value that is a single-key map, the value is replaced with the value of the + * macro named by the key, with any use of `(?#)` in the macro text replaced with the text of the + * value of the key, surrounded by a non-capturing group (`(?:)`). For example: + * + * The `beginPattern` and `endPattern` Properties + * A rule can have a `beginPattern` or `endPattern` property whose value is a reference to another + * rule (e.g. `#other-rule`). The `beginPattern` property is replaced as follows: + * + * my-rule: + * beginPattern: '#other-rule' + * + * would be transformed to + * + * my-rule: + * begin: '(?#other-rule)' + * beginCaptures: + * '0': + * patterns: + * - include: '#other-rule' + * + * An `endPattern` property is transformed similary. + * + * macros: + * repeat: '(?#)*' + * repository: + * multi-letter: + * match: + * repeat: '[A-Za-z]' + * name: scope.multi-letter + * + * would be transformed to + * + * repository: + * multi-letter: + * match: '(?:[A-Za-z])*' + * name: scope.multi-letter + */ + macros?: Record; + + patterns: Array>; + repository?: Record>; +} + +export interface Pattern { + include?: string; + match?: MatchType; + begin?: MatchType; + end?: MatchType; + while?: MatchType; + captures?: Record; + beginCaptures?: Record; + endCaptures?: Record; + patterns?: Array>; + beginPattern?: string; + endPattern?: string; +} + +export interface PatternCapture { + name?: string; + patterns?: Pattern[]; +} + +export type ExtendedMatchType = string | Record; diff --git a/extensions/ql-vscode/gulpfile.ts/textmate.ts b/extensions/ql-vscode/gulpfile.ts/textmate.ts index 25b94c5e6..0b7d2b88f 100644 --- a/extensions/ql-vscode/gulpfile.ts/textmate.ts +++ b/extensions/ql-vscode/gulpfile.ts/textmate.ts @@ -3,6 +3,12 @@ import { load } from "js-yaml"; import { obj } from "through2"; import PluginError from "plugin-error"; import type Vinyl from "vinyl"; +import type { + ExtendedMatchType, + ExtendedTextmateGrammar, + Pattern, + TextmateGrammar, +} from "./textmate-grammar"; /** * Replaces all rule references with the match pattern of the referenced rule. @@ -34,7 +40,9 @@ function replaceReferencesWithStrings( * @param yaml The root of the YAML document. * @returns A map from macro name to replacement text. */ -function gatherMacros(yaml: any): Map { +function gatherMacros( + yaml: ExtendedTextmateGrammar, +): Map { const macros = new Map(); for (const key in yaml.macros) { macros.set(key, yaml.macros[key]); @@ -51,7 +59,7 @@ function gatherMacros(yaml: any): Map { * @returns The match text for the rule. This is either the value of the rule's `match` property, * or the disjunction of the match text of all of the other rules `include`d by this rule. */ -function getNodeMatchText(rule: any): string { +function getNodeMatchText(rule: Pattern): string { if (rule.match !== undefined) { // For a match string, just use that string as the replacement. return rule.match; @@ -78,7 +86,7 @@ function getNodeMatchText(rule: any): string { * @returns A map whose keys are the names of rules, and whose values are the corresponding match * text of each rule. */ -function gatherMatchTextForRules(yaml: any): Map { +function gatherMatchTextForRules(yaml: TextmateGrammar): Map { const replacements = new Map(); for (const key in yaml.repository) { const node = yaml.repository[key]; @@ -94,9 +102,14 @@ function gatherMatchTextForRules(yaml: any): Map { * @param yaml The root of the YAML document. * @param action Callback to invoke on each rule. */ -function visitAllRulesInFile(yaml: any, action: (rule: any) => void) { +function visitAllRulesInFile( + yaml: ExtendedTextmateGrammar, + action: (rule: Pattern) => void, +) { visitAllRulesInRuleMap(yaml.patterns, action); - visitAllRulesInRuleMap(yaml.repository, action); + if (yaml.repository) { + visitAllRulesInRuleMap(Object.values(yaml.repository), action); + } } /** @@ -107,9 +120,11 @@ function visitAllRulesInFile(yaml: any, action: (rule: any) => void) { * @param ruleMap The map or array of rules to visit. * @param action Callback to invoke on each rule. */ -function visitAllRulesInRuleMap(ruleMap: any, action: (rule: any) => void) { - for (const key in ruleMap) { - const rule = ruleMap[key]; +function visitAllRulesInRuleMap( + ruleMap: Array>, + action: (rule: Pattern) => void, +) { + for (const rule of ruleMap) { if (typeof rule === "object") { action(rule); if (rule.patterns !== undefined) { @@ -125,16 +140,22 @@ function visitAllRulesInRuleMap(ruleMap: any, action: (rule: any) => void) { * @param rule The rule whose matches are to be transformed. * @param action The transformation to make on each match pattern. */ -function visitAllMatchesInRule(rule: any, action: (match: any) => any) { +function visitAllMatchesInRule(rule: Pattern, action: (match: T) => T) { for (const key in rule) { switch (key) { case "begin": case "end": case "match": - case "while": - rule[key] = action(rule[key]); - break; + case "while": { + const ruleElement = rule[key]; + if (!ruleElement) { + continue; + } + + rule[key] = action(ruleElement); + break; + } default: break; } @@ -148,14 +169,17 @@ function visitAllMatchesInRule(rule: any, action: (match: any) => any) { * @param rule Rule to be transformed. * @param key Base key of the property to be transformed. */ -function expandPatternMatchProperties(rule: any, key: "begin" | "end") { - const patternKey = `${key}Pattern`; - const capturesKey = `${key}Captures`; +function expandPatternMatchProperties( + rule: Pattern, + key: "begin" | "end", +) { + const patternKey = `${key}Pattern` as const; + const capturesKey = `${key}Captures` as const; const pattern = rule[patternKey]; if (pattern !== undefined) { const patterns: string[] = Array.isArray(pattern) ? pattern : [pattern]; - rule[key] = patterns.map((p) => `((?${p}))`).join("|"); - const captures: { [index: string]: any } = {}; + rule[key] = patterns.map((p) => `((?${p}))`).join("|") as T; + const captures: Pattern["captures"] = {}; for (const patternIndex in patterns) { captures[(Number(patternIndex) + 1).toString()] = { patterns: [ @@ -175,7 +199,7 @@ function expandPatternMatchProperties(rule: any, key: "begin" | "end") { * * @param yaml The root of the YAML document. */ -function transformFile(yaml: any) { +function transformFile(yaml: ExtendedTextmateGrammar) { const macros = gatherMacros(yaml); visitAllRulesInFile(yaml, (rule) => { expandPatternMatchProperties(rule, "begin"); @@ -198,24 +222,29 @@ function transformFile(yaml: any) { yaml.macros = undefined; - const replacements = gatherMatchTextForRules(yaml); + // We have removed all object match properties, so we don't have an extended match type anymore. + const macrolessYaml = yaml as ExtendedTextmateGrammar; + + const replacements = gatherMatchTextForRules(macrolessYaml); // Expand references in matches. - visitAllRulesInFile(yaml, (rule) => { + visitAllRulesInFile(macrolessYaml, (rule) => { visitAllMatchesInRule(rule, (match) => { return replaceReferencesWithStrings(match, replacements); }); }); - if (yaml.regexOptions !== undefined) { - const regexOptions = `(?${yaml.regexOptions})`; - visitAllRulesInFile(yaml, (rule) => { + if (macrolessYaml.regexOptions !== undefined) { + const regexOptions = `(?${macrolessYaml.regexOptions})`; + visitAllRulesInFile(macrolessYaml, (rule) => { visitAllMatchesInRule(rule, (match) => { return regexOptions + match; }); }); - yaml.regexOptions = undefined; + macrolessYaml.regexOptions = undefined; } + + return macrolessYaml; } export function transpileTextMateGrammar() { @@ -230,8 +259,8 @@ export function transpileTextMateGrammar() { } else if (file.isBuffer()) { const buf: Buffer = file.contents; const yamlText: string = buf.toString("utf8"); - const jsonData: any = load(yamlText); - transformFile(jsonData); + const yamlData = load(yamlText) as TextmateGrammar; + const jsonData = transformFile(yamlData); file.contents = Buffer.from(JSON.stringify(jsonData, null, 2), "utf8"); file.extname = ".json";