Extract textmate generator to helper library (#380)
This commit is contained in:
Nick Guerrera
GitHub
Родитель
7034d33f78
Коммит
672cb699fa
|
|
@ -1,6 +1,23 @@
|
|||
{
|
||||
"name": "adl-vscode",
|
||||
"entries": [
|
||||
{
|
||||
"version": "0.4.2",
|
||||
"tag": "adl-vscode_v0.4.2",
|
||||
"date": "Wed, 24 Mar 2021 18:40:21 GMT",
|
||||
"comments": {
|
||||
"patch": [
|
||||
{
|
||||
"comment": "Extract textmate generator to helper library"
|
||||
}
|
||||
],
|
||||
"dependency": [
|
||||
{
|
||||
"comment": "Updating dependency \"@azure-tools/tmlanguage-generator\" from `0.1.0` to `0.1.1`"
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"version": "0.4.1",
|
||||
"tag": "adl-vscode_v0.4.1",
|
||||
|
|
|
|||
|
|
@ -1,6 +1,13 @@
|
|||
# Change Log - adl-vscode
|
||||
|
||||
This log was last generated on Tue, 23 Mar 2021 01:06:29 GMT and should not be manually modified.
|
||||
This log was last generated on Wed, 24 Mar 2021 18:40:21 GMT and should not be manually modified.
|
||||
|
||||
## 0.4.2
|
||||
Wed, 24 Mar 2021 18:40:21 GMT
|
||||
|
||||
### Patches
|
||||
|
||||
- Extract textmate generator to helper library
|
||||
|
||||
## 0.4.1
|
||||
Tue, 23 Mar 2021 01:06:29 GMT
|
||||
|
|
|
|||
|
|
@ -1,25 +1,15 @@
|
|||
// TextMate-based syntax highlighting is implemented in this file.
|
||||
// adl.tmLanguage.json is generated by running this script.
|
||||
|
||||
import fs from "fs";
|
||||
import { promisify } from "util";
|
||||
import { loadWASM, OnigRegExp } from "onigasm";
|
||||
import * as tm from "@azure-tools/tmlanguage-generator";
|
||||
import fs from "fs/promises";
|
||||
|
||||
const readFile = promisify(fs.readFile);
|
||||
const writeFile = promisify(fs.writeFile);
|
||||
type IncludeRule = tm.IncludeRule<ADLScope>;
|
||||
type BeginEndRule = tm.BeginEndRule<ADLScope>;
|
||||
type MatchRule = tm.MatchRule<ADLScope>;
|
||||
type Grammar = tm.Grammar<ADLScope>;
|
||||
|
||||
const schema = "https://raw.githubusercontent.com/martinring/tmlanguage/master/tmlanguage.json";
|
||||
|
||||
// Special scope that indicates a larger construct that doesn't get a single color.
|
||||
// Expanded to meta.<key>.adl when we emit.
|
||||
const meta = "<meta>";
|
||||
|
||||
/**
|
||||
* The TextMate scope that gets assigned to a match and colored by a theme.
|
||||
* See https://macromates.com/manual/en/language_grammars#naming_conventions
|
||||
*/
|
||||
type Scope =
|
||||
| typeof meta
|
||||
type ADLScope =
|
||||
| "comment.block.adl"
|
||||
| "comment.line.double-slash.adl"
|
||||
| "constant.character.escape.adl"
|
||||
|
|
@ -31,43 +21,7 @@ type Scope =
|
|||
| "string.quoted.double.adl"
|
||||
| "variable.name.adl";
|
||||
|
||||
interface RuleKey {
|
||||
/** Rule's unique key through which identifies the rule in the repository. */
|
||||
key: string;
|
||||
}
|
||||
|
||||
interface RuleScope {
|
||||
scope: Scope;
|
||||
}
|
||||
|
||||
interface RulePatterns {
|
||||
patterns: Rule[];
|
||||
}
|
||||
|
||||
type Captures = Record<string, RuleScope | RulePatterns>;
|
||||
type Rule = MatchRule | BeginEndRule | IncludeRule;
|
||||
|
||||
interface MatchRule extends RuleScope, RuleKey {
|
||||
match: string;
|
||||
captures?: Captures;
|
||||
}
|
||||
|
||||
interface BeginEndRule extends RuleKey, RuleScope, Partial<RulePatterns> {
|
||||
begin: string;
|
||||
end: string;
|
||||
beginCaptures?: Captures;
|
||||
endCaptures?: Captures;
|
||||
}
|
||||
|
||||
interface IncludeRule extends RuleKey, RulePatterns {}
|
||||
|
||||
interface Grammar extends RulePatterns {
|
||||
$schema: typeof schema;
|
||||
name: string;
|
||||
scopeName: string;
|
||||
fileTypes: string[];
|
||||
}
|
||||
|
||||
const meta: typeof tm.meta = tm.meta;
|
||||
const identifierStart = "[_$[:alpha:]]";
|
||||
const identifierContinue = "[_$[:alnum:]]";
|
||||
const beforeIdentifier = `(?=${identifierStart})`;
|
||||
|
|
@ -309,130 +263,23 @@ const operationStatement: BeginEndRule = {
|
|||
// expressions color acceptably as unclassified punctuation around those we do
|
||||
// handle here.
|
||||
expression.patterns = [token, parenthesizedExpression, modelExpression, identifierExpression];
|
||||
|
||||
statement.patterns = [token, decorator, modelStatement, namespaceStatement, operationStatement];
|
||||
|
||||
const grammar: Grammar = {
|
||||
$schema: schema,
|
||||
$schema: tm.schema,
|
||||
name: "ADL",
|
||||
scopeName: "source.adl",
|
||||
fileTypes: [".adl"],
|
||||
patterns: [statement],
|
||||
};
|
||||
|
||||
/** Entry point, write the grammar to disk. */
|
||||
async function main() {
|
||||
const onigasm = await readFile("node_modules/onigasm/lib/onigasm.wasm");
|
||||
await loadWASM(onigasm.buffer);
|
||||
const filePath = "./dist/adl.tmlanguage.json";
|
||||
const json = emit(grammar);
|
||||
await writeFile(filePath, json);
|
||||
const json = await tm.emitJSON(grammar);
|
||||
await fs.writeFile(filePath, json);
|
||||
}
|
||||
|
||||
/** Emit the grammar to a JSON string matching tmlanguage.json schema. */
|
||||
function emit(grammar: Grammar): string {
|
||||
const indent = 2;
|
||||
const processed = processGrammar(grammar);
|
||||
return JSON.stringify(processed, undefined, indent);
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert the grammar from our more convenient representation to the
|
||||
* tmlanguage.json schema. Perform some validation in the process.
|
||||
*/
|
||||
function processGrammar(grammar: Grammar): any {
|
||||
// key is rule.key, value is [unprocessed rule, processed rule]. unprocessed
|
||||
// rule is used for its identity to check for duplicates and deal with cycles.
|
||||
const repository = new Map<string, [Rule, any]>();
|
||||
const output = processNode(grammar);
|
||||
output.repository = processRepository();
|
||||
return output;
|
||||
|
||||
function processNode(node: any): any {
|
||||
if (typeof node !== "object") {
|
||||
return node;
|
||||
}
|
||||
if (Array.isArray(node)) {
|
||||
return node.map(processNode);
|
||||
}
|
||||
const output: any = {};
|
||||
for (const key in node) {
|
||||
const value = node[key];
|
||||
switch (key) {
|
||||
case "key":
|
||||
// Drop it. It was used to place the node in the repository, and does
|
||||
// not need to be retained on the node in the final structure.
|
||||
break;
|
||||
case "scope":
|
||||
// tmlanguage uses "name" confusingly for scope. We avoid "name" which
|
||||
// can be confused with the repository key.
|
||||
output.name = value === meta ? `meta.${node.key}.adl` : value;
|
||||
break;
|
||||
case "begin":
|
||||
case "end":
|
||||
case "match":
|
||||
validateRegexp(value, node, key);
|
||||
output[key] = value;
|
||||
break;
|
||||
case "patterns":
|
||||
output[key] = processPatterns(value);
|
||||
break;
|
||||
default:
|
||||
output[key] = processNode(value);
|
||||
break;
|
||||
}
|
||||
}
|
||||
return output;
|
||||
}
|
||||
|
||||
function processPatterns(rules: Rule[]) {
|
||||
for (const rule of rules) {
|
||||
if (!repository.has(rule.key)) {
|
||||
// put placeholder first to prevent cycles
|
||||
const entry: [Rule, any] = [rule, undefined];
|
||||
repository.set(rule.key, entry);
|
||||
// fill placeholder with processed node.
|
||||
entry[1] = processNode(rule);
|
||||
} else if (repository.get(rule.key)![0] !== rule) {
|
||||
throw new Error("Duplicate key: " + rule.key);
|
||||
}
|
||||
}
|
||||
|
||||
return rules.map((r) => ({ include: `#${r.key}` }));
|
||||
}
|
||||
|
||||
function processRepository() {
|
||||
const output: any = {};
|
||||
for (const key of [...repository.keys()].sort()) {
|
||||
output[key] = repository.get(key)![1];
|
||||
}
|
||||
return output;
|
||||
}
|
||||
|
||||
function validateRegexp(regexp: string, node: any, prop: string) {
|
||||
try {
|
||||
new OnigRegExp(regexp).testSync("");
|
||||
} catch (err) {
|
||||
let message: string = err.message;
|
||||
if (/^[0-9,]+/.test(message)) {
|
||||
// Work around for https://github.com/NeekSandhu/onigasm/issues/26
|
||||
const array = new Uint8Array(message.split(",").map((s: string) => Number(s)));
|
||||
const buffer = Buffer.from(array);
|
||||
message = buffer.toString("utf-8");
|
||||
}
|
||||
console.error(`Error: Bad regex: ${JSON.stringify({ [prop]: regexp })}`);
|
||||
console.error(`Error: ${message}`);
|
||||
console.error();
|
||||
console.error("Context:");
|
||||
console.dir(node);
|
||||
console.error();
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
main()
|
||||
.then()
|
||||
.catch((err) => {
|
||||
console.log(err.stack);
|
||||
});
|
||||
main().catch((err) => {
|
||||
console.log(err.stack);
|
||||
process.exit(1);
|
||||
});
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@
|
|||
"url": "git+https://github.com/azure/adl"
|
||||
},
|
||||
"publisher": "Microsoft",
|
||||
"version": "0.4.1",
|
||||
"version": "0.4.2",
|
||||
"engines": {
|
||||
"vscode": "^1.53.0"
|
||||
},
|
||||
|
|
@ -24,7 +24,7 @@
|
|||
"ThirdPartyNotices.txt"
|
||||
],
|
||||
"scripts": {
|
||||
"build": "npm run compile && npm run generate-tmlanguage && npm run rollup && npm run generate-third-party-notices && npm run package-vsix",
|
||||
"build": "npm run compile && npm run rollup && npm run generate-tmlanguage && npm run generate-third-party-notices && npm run package-vsix",
|
||||
"prepare": "npm run build",
|
||||
"compile": "tsc -p .",
|
||||
"watch": "tsc -p . --watch",
|
||||
|
|
@ -42,7 +42,7 @@
|
|||
"@rollup/plugin-node-resolve": "^11.2.0",
|
||||
"@types/node": "14.0.27",
|
||||
"@types/vscode": "^1.53.0",
|
||||
"onigasm": "~2.2.5",
|
||||
"@azure-tools/tmlanguage-generator": "0.1.1",
|
||||
"rollup": "^2.41.4",
|
||||
"typescript": "~4.1.5",
|
||||
"vsce": "^1.85.1",
|
||||
|
|
|
|||
|
|
@ -0,0 +1,17 @@
|
|||
{
|
||||
"name": "@azure-tools/tmlanguage-generator",
|
||||
"entries": [
|
||||
{
|
||||
"version": "0.1.1",
|
||||
"tag": "@azure-tools/tmlanguage-generator_v0.1.1",
|
||||
"date": "Wed, 24 Mar 2021 18:40:21 GMT",
|
||||
"comments": {
|
||||
"patch": [
|
||||
{
|
||||
"comment": "Initial release"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
|
|
@ -0,0 +1,11 @@
|
|||
# Change Log - @azure-tools/textmate
|
||||
|
||||
This log was last generated on Wed, 24 Mar 2021 18:40:21 GMT and should not be manually modified.
|
||||
|
||||
## 0.1.1
|
||||
Wed, 24 Mar 2021 18:40:21 GMT
|
||||
|
||||
### Patches
|
||||
|
||||
- Initial release
|
||||
|
||||
|
|
@ -0,0 +1,21 @@
|
|||
MIT License
|
||||
|
||||
Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all
|
||||
copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
SOFTWARE
|
||||
|
|
@ -0,0 +1 @@
|
|||
# Helper library for generating TextMate syntax highlighting files
|
||||
|
|
@ -0,0 +1,173 @@
|
|||
import { loadWASM, OnigRegExp } from "onigasm";
|
||||
import { readFile } from "fs/promises";
|
||||
import { resolve } from "path";
|
||||
|
||||
export const schema =
|
||||
"https://raw.githubusercontent.com/martinring/tmlanguage/master/tmlanguage.json";
|
||||
|
||||
/**
|
||||
* Special scope that indicates a larger construct that doesn't get a single color.
|
||||
* Expanded to meta.<key>.<grammar name> during emit.
|
||||
*/
|
||||
export const meta = Symbol("meta");
|
||||
|
||||
export interface RuleKey {
|
||||
/** Rule's unique key through which identifies the rule in the repository. */
|
||||
key: string;
|
||||
}
|
||||
|
||||
export interface RuleScope<Scope extends string = string> {
|
||||
/**
|
||||
* The TextMate scope that gets assigned to a match and colored by a theme.
|
||||
* See https://macromates.com/manual/en/language_grammars#naming_conventions
|
||||
*/
|
||||
scope: Scope | typeof meta;
|
||||
}
|
||||
|
||||
export interface RulePatterns<Scope extends string = string> {
|
||||
patterns: Rule<Scope>[];
|
||||
}
|
||||
|
||||
export type Captures<Scope extends string = string> = Record<
|
||||
string,
|
||||
RuleScope<Scope> | RulePatterns<Scope>
|
||||
>;
|
||||
|
||||
export type Rule<Scope extends string = string> =
|
||||
| MatchRule<Scope>
|
||||
| BeginEndRule<Scope>
|
||||
| IncludeRule<Scope>;
|
||||
|
||||
export interface MatchRule<Scope extends string = string> extends RuleScope<Scope>, RuleKey {
|
||||
match: string;
|
||||
captures?: Captures<Scope>;
|
||||
}
|
||||
|
||||
export interface BeginEndRule<Scope extends string = string>
|
||||
extends RuleKey,
|
||||
RuleScope<Scope>,
|
||||
Partial<RulePatterns<Scope>> {
|
||||
begin: string;
|
||||
end: string;
|
||||
beginCaptures?: Captures<Scope>;
|
||||
endCaptures?: Captures<Scope>;
|
||||
}
|
||||
|
||||
export interface IncludeRule<Scope extends string = string> extends RuleKey, RulePatterns<Scope> {}
|
||||
|
||||
export interface Grammar<Scope extends string = string> extends RulePatterns<Scope> {
|
||||
$schema: typeof schema;
|
||||
name: string;
|
||||
scopeName: string;
|
||||
fileTypes: string[];
|
||||
}
|
||||
|
||||
let initialized = false;
|
||||
async function initialize() {
|
||||
if (!initialized) {
|
||||
const path = resolve(__dirname, "../node_modules/onigasm/lib/onigasm.wasm");
|
||||
const wasm = await readFile(path);
|
||||
await loadWASM(wasm.buffer);
|
||||
initialized = true;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Emit the given grammar to JSON.
|
||||
*/
|
||||
export async function emitJSON(grammar: Grammar): Promise<string> {
|
||||
await initialize();
|
||||
const indent = 2;
|
||||
const processed = processGrammar(grammar);
|
||||
return JSON.stringify(processed, undefined, indent);
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert the grammar from our more convenient representation to the
|
||||
* tmlanguage.json schema. Perform some validation in the process.
|
||||
*/
|
||||
function processGrammar(grammar: Grammar): any {
|
||||
// key is rule.key, value is [unprocessed rule, processed rule]. unprocessed
|
||||
// rule is used for its identity to check for duplicates and deal with cycles.
|
||||
const repository = new Map<string, [Rule, any]>();
|
||||
const output = processNode(grammar);
|
||||
output.repository = processRepository();
|
||||
return output;
|
||||
|
||||
function processNode(node: any): any {
|
||||
if (typeof node !== "object") {
|
||||
return node;
|
||||
}
|
||||
if (Array.isArray(node)) {
|
||||
return node.map(processNode);
|
||||
}
|
||||
const output: any = {};
|
||||
for (const key in node) {
|
||||
const value = node[key];
|
||||
switch (key) {
|
||||
case "key":
|
||||
// Drop it. It was used to place the node in the repository, and does
|
||||
// not need to be retained on the node in the final structure.
|
||||
break;
|
||||
case "scope":
|
||||
// tmlanguage uses "name" confusingly for scope. We avoid "name" which
|
||||
// can be confused with the repository key.
|
||||
output.name = value === meta ? `meta.${node.key}.adl` : value;
|
||||
break;
|
||||
case "begin":
|
||||
case "end":
|
||||
case "match":
|
||||
validateRegexp(value, node, key);
|
||||
output[key] = value;
|
||||
break;
|
||||
case "patterns":
|
||||
output[key] = processPatterns(value);
|
||||
break;
|
||||
default:
|
||||
output[key] = processNode(value);
|
||||
break;
|
||||
}
|
||||
}
|
||||
return output;
|
||||
}
|
||||
|
||||
function processPatterns(rules: Rule[]) {
|
||||
for (const rule of rules) {
|
||||
if (!repository.has(rule.key)) {
|
||||
// put placeholder first to prevent cycles
|
||||
const entry: [Rule, any] = [rule, undefined];
|
||||
repository.set(rule.key, entry);
|
||||
// fill placeholder with processed node.
|
||||
entry[1] = processNode(rule);
|
||||
} else if (repository.get(rule.key)![0] !== rule) {
|
||||
throw new Error("Duplicate key: " + rule.key);
|
||||
}
|
||||
}
|
||||
|
||||
return rules.map((r) => ({ include: `#${r.key}` }));
|
||||
}
|
||||
|
||||
function processRepository() {
|
||||
const output: any = {};
|
||||
for (const key of [...repository.keys()].sort()) {
|
||||
output[key] = repository.get(key)![1];
|
||||
}
|
||||
return output;
|
||||
}
|
||||
|
||||
function validateRegexp(regexp: string, node: any, prop: string) {
|
||||
try {
|
||||
new OnigRegExp(regexp).testSync("");
|
||||
} catch (err) {
|
||||
if (/^[0-9,]+/.test(err.message)) {
|
||||
// Work around for https://github.com/NeekSandhu/onigasm/issues/26
|
||||
const array = new Uint8Array(err.message.split(",").map((s: string) => Number(s)));
|
||||
const buffer = Buffer.from(array);
|
||||
err = new Error(buffer.toString("utf-8"));
|
||||
}
|
||||
console.error(`Error: Bad regex: ${JSON.stringify({ [prop]: regexp })} in:`);
|
||||
console.error(node);
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,40 @@
|
|||
{
|
||||
"name": "@azure-tools/tmlanguage-generator",
|
||||
"version": "0.1.1",
|
||||
"author": "Microsoft Corporation",
|
||||
"description": "Helper library to generate TextMate syntax highlighting tmlanguage files.",
|
||||
"homepage": "https://github.com/Azure/adl/tree/master/packages/textmate",
|
||||
"readme": "https://github.com/Azure/adl/blob/master/packages/textmate/REAMDE.md",
|
||||
"keywords": [
|
||||
"textmate",
|
||||
"tmlanguage"
|
||||
],
|
||||
"license": "MIT",
|
||||
"repository": {
|
||||
"type": "git",
|
||||
"url": "git+https://github.com/Azure/adl.git"
|
||||
},
|
||||
"bugs": {
|
||||
"url": "https://github.com/Azure/adl/issues"
|
||||
},
|
||||
"main": "dist/index.js",
|
||||
"engines": {
|
||||
"node": ">=14.0.0"
|
||||
},
|
||||
"scripts": {
|
||||
"build": "tsc -p .",
|
||||
"prepare": "npm run build",
|
||||
"watch": "tsc -p . --watch",
|
||||
"check-format": "prettier --list-different --config ../../.prettierrc.json --ignore-path ../../.prettierignore \"**/*.{ts,js,json}\"",
|
||||
"format": "prettier --write --config ../../.prettierrc.json --ignore-path ../../.prettierignore \"**/*.{ts,js,json}\""
|
||||
},
|
||||
"files": [
|
||||
"dist/**"
|
||||
],
|
||||
"dependencies": {
|
||||
"onigasm": "~2.2.5"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/node": "14.0.27"
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,10 @@
|
|||
{
|
||||
"extends": "../tsconfig.json",
|
||||
"compilerOptions": {
|
||||
"outDir": "dist",
|
||||
"rootDir": ".",
|
||||
"types": ["node"]
|
||||
},
|
||||
"include": ["./**/*.ts"],
|
||||
"exclude": ["dist", "node_modules", "**/*.d.ts"]
|
||||
}
|
||||
Загрузка…
Ссылка в новой задаче