microsoft
/
vscode-textmate
зеркало из https://github.com/microsoft/vscode-textmate.git
1
0
Форкнуть 0
Код Задачи Пакеты Проекты Релизы Вики Активность

Merge pull request #14 from Microsoft/kieferrm/support-while 

Support for grammars with begin/while rules
This commit is contained in:
Alexandru Dima 2016-05-23 10:58:17 +02:00
Родитель 214c326136 2874c53bae
Коммит b14ff75250
7 изменённых файлов: 261 добавлений и 22 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

1
release/main.d.ts поставляемый
Просмотреть файл

@ -74,6 +74,7 @@ export interface StackElement {
ruleId: number;
enterPos: number;
endRule: string;
whileRule: string;
scopeName: string;
contentName: string;

118
release/main.js
Просмотреть файл

@ -767,6 +767,60 @@ var BeginEndRule = (function (_super) {
return BeginEndRule;
}(Rule));
exports.BeginEndRule = BeginEndRule;
var BeginWhileRule = (function (_super) {
__extends(BeginWhileRule, _super);
function BeginWhileRule(id, name, contentName, begin, beginCaptures, _while, patterns) {
_super.call(this, id, name, contentName);
this._begin = new RegExpSource(begin, this.id);
this.beginCaptures = beginCaptures;
this._while = new RegExpSource(_while, -2);
this.whileHasBackReferences = this._while.hasBackReferences;
this.patterns = patterns.patterns;
this.hasMissingPatterns = patterns.hasMissingPatterns;
this._cachedCompiledPatterns = null;
this._cachedCompiledWhilePatterns = null;
}
BeginWhileRule.prototype.getWhileWithResolvedBackReferences = function (lineText, captureIndices) {
return this._while.resolveBackReferences(lineText, captureIndices);
};
BeginWhileRule.prototype.collectPatternsRecursive = function (grammar, out, isFirst) {
if (isFirst) {
var i = void 0, len = void 0, rule = void 0;
for (i = 0, len = this.patterns.length; i < len; i++) {
rule = grammar.getRule(this.patterns[i]);
rule.collectPatternsRecursive(grammar, out, false);
}
}
else {
out.push(this._begin);
}
};
BeginWhileRule.prototype.compile = function (grammar, endRegexSource, allowA, allowG) {
this._precompile(grammar);
return this._cachedCompiledPatterns.compile(grammar, allowA, allowG);
};
BeginWhileRule.prototype._precompile = function (grammar) {
if (!this._cachedCompiledPatterns) {
this._cachedCompiledPatterns = new RegExpSourceList();
this.collectPatternsRecursive(grammar, this._cachedCompiledPatterns, true);
}
};
BeginWhileRule.prototype.compileWhile = function (grammar, endRegexSource, allowA, allowG) {
this._precompileWhile(grammar);
if (this._while.hasBackReferences) {
this._cachedCompiledWhilePatterns.setSource(0, endRegexSource);
}
return this._cachedCompiledWhilePatterns.compile(grammar, allowA, allowG);
};
BeginWhileRule.prototype._precompileWhile = function (grammar) {
if (!this._cachedCompiledWhilePatterns) {
this._cachedCompiledWhilePatterns = new RegExpSourceList();
this._cachedCompiledWhilePatterns.push(this._while.hasBackReferences ? this._while.clone() : this._while);
}
};
return BeginWhileRule;
}(Rule));
exports.BeginWhileRule = BeginWhileRule;
var RuleFactory = (function () {
function RuleFactory() {
}
@ -788,6 +842,9 @@ var RuleFactory = (function () {
}
return new IncludeOnlyRule(desc.id, desc.name, desc.contentName, RuleFactory._compilePatterns(desc.patterns, helper, repository));
}
if (desc.while) {
return new BeginWhileRule(desc.id, desc.name, desc.contentName, desc.begin, RuleFactory._compileCaptures(desc.beginCaptures || desc.captures, helper, repository), desc.while, RuleFactory._compilePatterns(desc.patterns, helper, repository));
}
return new BeginEndRule(desc.id, desc.name, desc.contentName, desc.begin, RuleFactory._compileCaptures(desc.beginCaptures || desc.captures, helper, repository), desc.end, RuleFactory._compileCaptures(desc.endCaptures || desc.captures, helper, repository), desc.applyEndPatternLast, RuleFactory._compilePatterns(desc.patterns, helper, repository));
});
}
@ -874,12 +931,7 @@ var RuleFactory = (function () {
if (patternId !== -1) {
rule = helper.getRule(patternId);
skipRule = false;
if (rule instanceof IncludeOnlyRule) {
if (rule.hasMissingPatterns && rule.patterns.length === 0) {
skipRule = true;
}
}
else if (rule instanceof BeginEndRule) {
if (rule instanceof IncludeOnlyRule || rule instanceof BeginEndRule || rule instanceof BeginWhileRule) {
if (rule.hasMissingPatterns && rule.patterns.length === 0) {
skipRule = true;
}
@ -1178,7 +1230,26 @@ function matchInjections(injections, grammar, lineText, isFirstLine, linePos, st
}
function matchRule(grammar, lineText, isFirstLine, linePos, stack, anchorPosition) {
var stackElement = stack[stack.length - 1];
var ruleScanner = grammar.getRule(stackElement.ruleId).compile(grammar, stackElement.endRule, isFirstLine, linePos === anchorPosition);
var rule = grammar.getRule(stackElement.ruleId);
if (rule instanceof rule_1.BeginWhileRule && stackElement.enterPos === -1) {
var ruleScanner_1 = rule.compileWhile(grammar, stackElement.endRule || stackElement.whileRule, isFirstLine, linePos === anchorPosition);
var r_1 = ruleScanner_1.scanner._findNextMatchSync(lineText, linePos);
var doNotContinue = {
captureIndices: null,
matchedRuleId: -3
};
if (r_1) {
var matchedRuleId = ruleScanner_1.rules[r_1.index];
if (matchedRuleId != -2) {
// we shouldn't end up here
return doNotContinue;
}
}
else {
return doNotContinue;
}
}
var ruleScanner = rule.compile(grammar, stackElement.endRule || stackElement.whileRule, isFirstLine, linePos === anchorPosition);
var r = ruleScanner.scanner._findNextMatchSync(lineText, linePos);
if (r) {
return {
@ -1232,7 +1303,7 @@ function _tokenizeString(grammar, lineText, isFirstLine, linePos, stack, lineTok
}
var captureIndices = r.captureIndices;
var matchedRuleId = r.matchedRuleId;
var hasAdvanced = (captureIndices[0].end > linePos);
var hasAdvanced = (captureIndices && captureIndices.length > 0) ? (captureIndices[0].end > linePos) : false;
if (matchedRuleId === -1) {
// We matched the `end` for this rule => pop it
var poppedRule = grammar.getRule(stackElement.ruleId);
@ -1250,12 +1321,17 @@ function _tokenizeString(grammar, lineText, isFirstLine, linePos, stack, lineTok
return false;
}
}
else if (matchedRuleId === -3) {
// A while clause failed
stack.pop();
return true;
}
else {
// We matched a rule!
var _rule = grammar.getRule(matchedRuleId);
lineTokens.produce(stack, captureIndices[0].start);
// push it on the stack rule
stack.push(new StackElement(matchedRuleId, linePos, null, _rule.getName(rule_1.getString(lineText), captureIndices), null));
stack.push(new StackElement(matchedRuleId, linePos, null, _rule.getName(rule_1.getString(lineText), captureIndices), null, null));
if (_rule instanceof rule_1.BeginEndRule) {
var pushedRule = _rule;
handleCaptures(grammar, lineText, isFirstLine, stack, lineTokens, pushedRule.beginCaptures, captureIndices);
@ -1274,6 +1350,24 @@ function _tokenizeString(grammar, lineText, isFirstLine, linePos, stack, lineTok
return false;
}
}
else if (_rule instanceof rule_1.BeginWhileRule) {
var pushedRule = _rule;
handleCaptures(grammar, lineText, isFirstLine, stack, lineTokens, pushedRule.beginCaptures, captureIndices);
lineTokens.produce(stack, captureIndices[0].end);
anchorPosition = captureIndices[0].end;
stack[stack.length - 1].contentName = pushedRule.getContentName(rule_1.getString(lineText), captureIndices);
if (pushedRule.whileHasBackReferences) {
stack[stack.length - 1].whileRule = pushedRule.getWhileWithResolvedBackReferences(rule_1.getString(lineText), captureIndices);
}
if (!hasAdvanced && stackElement.ruleId === stack[stack.length - 1].ruleId) {
// Grammar pushed the same rule without advancing
console.error('Grammar is in an endless loop - case 2');
stack.pop();
lineTokens.produce(stack, lineLength);
linePos = lineLength;
return false;
}
}
else {
var matchingRule = _rule;
handleCaptures(grammar, lineText, isFirstLine, stack, lineTokens, matchingRule.captures, captureIndices);
@ -1301,15 +1395,17 @@ function _tokenizeString(grammar, lineText, isFirstLine, linePos, stack, lineTok
}
}
var StackElement = (function () {
function StackElement(ruleId, enterPos, endRule, scopeName, contentName) {
function StackElement(ruleId, enterPos, endRule, scopeName, contentName, whileRule) {
if (whileRule === void 0) { whileRule = null; }
this.ruleId = ruleId;
this.enterPos = enterPos;
this.endRule = endRule;
this.scopeName = scopeName;
this.contentName = contentName;
this.whileRule = whileRule;
}
StackElement.prototype.clone = function () {
return new StackElement(this.ruleId, this.enterPos, this.endRule, this.scopeName, this.contentName);
return new StackElement(this.ruleId, this.enterPos, this.endRule, this.scopeName, this.contentName, this.whileRule);
};
StackElement.prototype.matches = function (scopeName) {
if (!this.scopeName) {

62
src/grammar.ts
Просмотреть файл

@ -5,7 +5,7 @@
import {clone} from './utils';
import {IRawGrammar, IRawRepository, IRawRule} from './types';
import {IRuleFactoryHelper, RuleFactory, Rule, CaptureRule, BeginEndRule, MatchRule, ICompiledRule, createOnigString, getString} from './rule';
import {IRuleFactoryHelper, RuleFactory, Rule, CaptureRule, BeginEndRule, BeginWhileRule, MatchRule, ICompiledRule, createOnigString, getString} from './rule';
import {IOnigCaptureIndex, IOnigNextMatchResult, OnigString} from 'oniguruma';
import {createMatcher, Matcher} from './matcher';
@ -381,7 +381,31 @@ interface IMatchResult {
function matchRule(grammar: Grammar, lineText: OnigString, isFirstLine: boolean, linePos: number, stack: StackElement[], anchorPosition:number): IMatchResult {
let stackElement = stack[stack.length - 1];
let ruleScanner = grammar.getRule(stackElement.ruleId).compile(grammar, stackElement.endRule, isFirstLine, linePos === anchorPosition);
let rule = grammar.getRule(stackElement.ruleId);
if (rule instanceof BeginWhileRule && stackElement.enterPos === -1) {
let ruleScanner = rule.compileWhile(grammar, stackElement.endRule || stackElement.whileRule, isFirstLine, linePos === anchorPosition);
let r = ruleScanner.scanner._findNextMatchSync(lineText, linePos);
let doNotContinue: IMatchResult = {
captureIndices: null,
matchedRuleId: -3
};
if (r) {
let matchedRuleId = ruleScanner.rules[r.index];
if (matchedRuleId != -2) {
// we shouldn't end up here
return doNotContinue;
}
} else {
return doNotContinue;
}
}
let ruleScanner = rule.compile(grammar, stackElement.endRule || stackElement.whileRule, isFirstLine, linePos === anchorPosition);
let r = ruleScanner.scanner._findNextMatchSync(lineText, linePos);
if (r) {
@ -450,7 +474,7 @@ function _tokenizeString(grammar: Grammar, lineText: OnigString, isFirstLine: bo
let captureIndices: IOnigCaptureIndex[] = r.captureIndices;
let matchedRuleId: number = r.matchedRuleId;
let hasAdvanced = (captureIndices[0].end > linePos);
let hasAdvanced = (captureIndices && captureIndices.length > 0) ? (captureIndices[0].end > linePos) : false;
if (matchedRuleId === -1) {
// We matched the `end` for this rule => pop it
@ -471,6 +495,10 @@ function _tokenizeString(grammar: Grammar, lineText: OnigString, isFirstLine: bo
linePos = lineLength;
return false;
}
} else if (matchedRuleId === -3) {
// A while clause failed
stack.pop();
return true;
} else {
// We matched a rule!
@ -479,7 +507,7 @@ function _tokenizeString(grammar: Grammar, lineText: OnigString, isFirstLine: bo
lineTokens.produce(stack, captureIndices[0].start);
// push it on the stack rule
stack.push(new StackElement(matchedRuleId, linePos, null, _rule.getName(getString(lineText), captureIndices), null));
stack.push(new StackElement(matchedRuleId, linePos, null, _rule.getName(getString(lineText), captureIndices), null, null));
if (_rule instanceof BeginEndRule) {
let pushedRule = <BeginEndRule>_rule;
@ -493,6 +521,26 @@ function _tokenizeString(grammar: Grammar, lineText: OnigString, isFirstLine: bo
stack[stack.length-1].endRule = pushedRule.getEndWithResolvedBackReferences(getString(lineText), captureIndices);
}
if (!hasAdvanced && stackElement.ruleId === stack[stack.length - 1].ruleId) {
// Grammar pushed the same rule without advancing
console.error('Grammar is in an endless loop - case 2');
stack.pop();
lineTokens.produce(stack, lineLength);
linePos = lineLength;
return false;
}
} else if (_rule instanceof BeginWhileRule) {
let pushedRule = <BeginWhileRule>_rule;
handleCaptures(grammar, lineText, isFirstLine, stack, lineTokens, pushedRule.beginCaptures, captureIndices);
lineTokens.produce(stack, captureIndices[0].end);
anchorPosition = captureIndices[0].end;
stack[stack.length - 1].contentName = pushedRule.getContentName(getString(lineText), captureIndices);
if (pushedRule.whileHasBackReferences) {
stack[stack.length - 1].whileRule = pushedRule.getWhileWithResolvedBackReferences(getString(lineText), captureIndices);
}
if (!hasAdvanced && stackElement.ruleId === stack[stack.length - 1].ruleId) {
// Grammar pushed the same rule without advancing
console.error('Grammar is in an endless loop - case 2');
@ -538,19 +586,21 @@ export class StackElement {
public endRule: string;
public scopeName: string;
public contentName: string;
public whileRule: string;
private scopeNameSegments: { [segment:string]:boolean };
constructor (ruleId:number, enterPos:number, endRule:string, scopeName:string, contentName:string) {
constructor(ruleId:number, enterPos:number, endRule:string, scopeName:string, contentName: string, whileRule: string = null) {
this.ruleId = ruleId;
this.enterPos = enterPos;
this.endRule = endRule;
this.scopeName = scopeName;
this.contentName = contentName;
this.whileRule = whileRule;
}
public clone(): StackElement {
return new StackElement(this.ruleId, this.enterPos, this.endRule, this.scopeName, this.contentName);
return new StackElement(this.ruleId, this.enterPos, this.endRule, this.scopeName, this.contentName, this.whileRule);
}
public matches(scopeName: string) : boolean {

88
src/rule.ts
Просмотреть файл

@ -508,6 +508,77 @@ export class BeginEndRule extends Rule {
}
}
export class BeginWhileRule extends Rule {
private _begin: RegExpSource;
public beginCaptures: CaptureRule[];
private _while: RegExpSource;
public whileHasBackReferences: boolean;
public hasMissingPatterns: boolean;
public patterns: number[];
private _cachedCompiledPatterns: RegExpSourceList;
private _cachedCompiledWhilePatterns: RegExpSourceList;
constructor(id: number, name: string, contentName: string, begin: string, beginCaptures: CaptureRule[], _while: string, patterns: ICompilePatternsResult) {
super(id, name, contentName);
this._begin = new RegExpSource(begin, this.id);
this.beginCaptures = beginCaptures;
this._while = new RegExpSource(_while, -2);
this.whileHasBackReferences = this._while.hasBackReferences;
this.patterns = patterns.patterns;
this.hasMissingPatterns = patterns.hasMissingPatterns;
this._cachedCompiledPatterns = null;
this._cachedCompiledWhilePatterns = null;
}
public getWhileWithResolvedBackReferences(lineText: string, captureIndices: IOnigCaptureIndex[]): string {
return this._while.resolveBackReferences(lineText, captureIndices);
}
public collectPatternsRecursive(grammar: IRuleRegistry, out: RegExpSourceList, isFirst: boolean) {
if (isFirst) {
let i: number,
len: number,
rule: Rule;
for (i = 0, len = this.patterns.length; i < len; i++) {
rule = grammar.getRule(this.patterns[i]);
rule.collectPatternsRecursive(grammar, out, false);
}
} else {
out.push(this._begin);
}
}
public compile(grammar: IRuleRegistry, endRegexSource: string, allowA: boolean, allowG: boolean): ICompiledRule {
this._precompile(grammar);
return this._cachedCompiledPatterns.compile(grammar, allowA, allowG);
}
private _precompile(grammar: IRuleRegistry): void {
if (!this._cachedCompiledPatterns) {
this._cachedCompiledPatterns = new RegExpSourceList();
this.collectPatternsRecursive(grammar, this._cachedCompiledPatterns, true);
}
}
public compileWhile(grammar: IRuleRegistry, endRegexSource: string, allowA: boolean, allowG: boolean): ICompiledRule {
this._precompileWhile(grammar);
if (this._while.hasBackReferences) {
this._cachedCompiledWhilePatterns.setSource(0, endRegexSource);
}
return this._cachedCompiledWhilePatterns.compile(grammar, allowA, allowG);
}
private _precompileWhile(grammar: IRuleRegistry): void {
if (!this._cachedCompiledWhilePatterns) {
this._cachedCompiledWhilePatterns = new RegExpSourceList();
this._cachedCompiledWhilePatterns.push(this._while.hasBackReferences ? this._while.clone() : this._while);
}
}
}
export class RuleFactory {
public static createCaptureRule(helper: IRuleFactoryHelper, name:string, contentName:string, retokenizeCapturedWithRuleId:number): CaptureRule {
@ -542,6 +613,17 @@ export class RuleFactory {
);
}
if (desc.while) {
return new BeginWhileRule(
desc.id,
desc.name,
desc.contentName,
desc.begin, RuleFactory._compileCaptures(desc.beginCaptures || desc.captures, helper, repository),
desc.while,
RuleFactory._compilePatterns(desc.patterns, helper, repository)
);
}
return new BeginEndRule(
desc.id,
desc.name,
@ -658,11 +740,7 @@ export class RuleFactory {
skipRule = false;
if (rule instanceof IncludeOnlyRule) {
if (rule.hasMissingPatterns && rule.patterns.length === 0) {
skipRule = true;
}
} else if (rule instanceof BeginEndRule) {
if (rule instanceof IncludeOnlyRule || rule instanceof BeginEndRule || rule instanceof BeginWhileRule) {
if (rule.hasMissingPatterns && rule.patterns.length === 0) {
skipRule = true;
}

1
src/types.ts
Просмотреть файл

@ -37,6 +37,7 @@ export interface IRawRule {
beginCaptures?: IRawCaptures;
end?:string;
endCaptures?: IRawCaptures;
while?:string;
patterns?: IRawRule[];
repository?: IRawRepository;

1
src/typings/main.d.ts поставляемый
Просмотреть файл

@ -74,6 +74,7 @@ export interface StackElement {
ruleId: number;
enterPos: number;
endRule: string;
whileRule: string;
scopeName: string;
contentName: string;

12
test-cases/suite1/tests.json
Просмотреть файл

@ -77,6 +77,18 @@
"grammarPath": "fixtures/markdown.plist",
"desc": "Nested repositories in Markdown",
"lines": [
{
"line": "This is a paragraph",
"tokens": [
{
"value": "This is a paragraph",
"scopes": [
"text.html.markdown",
"meta.paragraph.markdown"
]
}
]
},
{
"line": "## This is *great* stuff",
"tokens": [