support begin/while rules

src/grammar.ts

@@ -5,7 +5,7 @@
 
 import {clone} from './utils';
 import {IRawGrammar, IRawRepository, IRawRule} from './types';
-import {IRuleFactoryHelper, RuleFactory, Rule, CaptureRule, BeginEndRule, MatchRule, ICompiledRule, createOnigString, getString} from './rule';
+import {IRuleFactoryHelper, RuleFactory, Rule, CaptureRule, BeginEndRule, BeginWhileRule, MatchRule, ICompiledRule, createOnigString, getString} from './rule';
 import {IOnigCaptureIndex, IOnigNextMatchResult, OnigString} from 'oniguruma';
 import {createMatcher, Matcher} from './matcher';
 
@@ -370,7 +370,31 @@ interface IMatchResult {
 
 function matchRule(grammar: Grammar, lineText: OnigString, isFirstLine: boolean, linePos: number, stack: StackElement[], anchorPosition:number): IMatchResult {
 	let stackElement = stack[stack.length - 1];
-	let ruleScanner = grammar.getRule(stackElement.ruleId).compile(grammar, stackElement.endRule, isFirstLine, linePos === anchorPosition);
+	let rule = grammar.getRule(stackElement.ruleId);
+
+	if (rule instanceof BeginWhileRule) {
+
+		let ruleScanner = rule.compileWhile(grammar, stackElement.endRule || stackElement.whileRule, isFirstLine, linePos === anchorPosition);
+		let r = ruleScanner.scanner._findNextMatchSync(lineText, linePos);
+
+		let doNotContinue: IMatchResult = {
+			captureIndices: null,
+			matchedRuleId: -3
+		};
+
+		if (r) {
+			let matchedRuleId = ruleScanner.rules[r.index];
+			if (matchedRuleId != -2) {
+				// we shouldn't end up here
+				return doNotContinue;
+			}
+		} else {
+			return doNotContinue;
+		}
+	}
+
+
+	let ruleScanner = rule.compile(grammar, stackElement.endRule || stackElement.whileRule, isFirstLine, linePos === anchorPosition);
 	let r = ruleScanner.scanner._findNextMatchSync(lineText, linePos);
 
 	if (r) {
@@ -439,7 +463,7 @@ function _tokenizeString(grammar: Grammar, lineText: OnigString, isFirstLine: bo
 		let captureIndices: IOnigCaptureIndex[] = r.captureIndices;
 		let matchedRuleId: number = r.matchedRuleId;
 
-		let hasAdvanced = (captureIndices[0].end > linePos);
+		let hasAdvanced = (captureIndices && captureIndices.length > 0) ? (captureIndices[0].end > linePos) : false;
 
 		if (matchedRuleId === -1) {
 			// We matched the `end` for this rule => pop it
@@ -460,6 +484,10 @@ function _tokenizeString(grammar: Grammar, lineText: OnigString, isFirstLine: bo
 				linePos = lineLength;
 				return false;
 			}
+		} else if (matchedRuleId === -3) {
+			// A while clause failed
+			stack.pop();
+			return true;
 
 		} else {
 			// We matched a rule!
@@ -468,7 +496,7 @@ function _tokenizeString(grammar: Grammar, lineText: OnigString, isFirstLine: bo
 			lineTokens.produce(stack, captureIndices[0].start);
 
 			// push it on the stack rule
-			stack.push(new StackElement(matchedRuleId, linePos, null, _rule.getName(getString(lineText), captureIndices), null));
+			stack.push(new StackElement(matchedRuleId, linePos, null, _rule.getName(getString(lineText), captureIndices), null, null));
 
 			if (_rule instanceof BeginEndRule) {
 				let pushedRule = <BeginEndRule>_rule;
@@ -482,6 +510,26 @@ function _tokenizeString(grammar: Grammar, lineText: OnigString, isFirstLine: bo
 					stack[stack.length-1].endRule = pushedRule.getEndWithResolvedBackReferences(getString(lineText), captureIndices);
 				}
 
+				if (!hasAdvanced && stackElement.ruleId === stack[stack.length - 1].ruleId) {
+					// Grammar pushed the same rule without advancing
+					console.error('Grammar is in an endless loop - case 2');
+					stack.pop();
+					lineTokens.produce(stack, lineLength);
+					linePos = lineLength;
+					return false;
+				}
+			} else if (_rule instanceof BeginWhileRule) {
+				let pushedRule = <BeginWhileRule>_rule;
+
+				handleCaptures(grammar, lineText, isFirstLine, stack, lineTokens, pushedRule.beginCaptures, captureIndices);
+				lineTokens.produce(stack, captureIndices[0].end);
+				anchorPosition = captureIndices[0].end;
+				stack[stack.length - 1].contentName = pushedRule.getContentName(getString(lineText), captureIndices);
+
+				if (pushedRule.whileHasBackReferences) {
+					stack[stack.length - 1].whileRule = pushedRule.getWhileWithResolvedBackReferences(getString(lineText), captureIndices);
+				}
+
 				if (!hasAdvanced && stackElement.ruleId === stack[stack.length - 1].ruleId) {
 					// Grammar pushed the same rule without advancing
 					console.error('Grammar is in an endless loop - case 2');
@@ -527,19 +575,21 @@ export class StackElement {
 	public endRule: string;
 	public scopeName: string;
 	public contentName: string;
+	public whileRule: string;
 
 	private scopeNameSegments: { [segment:string]:boolean };
 
-	constructor (ruleId:number, enterPos:number, endRule:string, scopeName:string, contentName:string) {
+	constructor(ruleId:number, enterPos:number, endRule:string, scopeName:string, contentName: string, whileRule: string = null) {
 		this.ruleId = ruleId;
 		this.enterPos = enterPos;
 		this.endRule = endRule;
 		this.scopeName = scopeName;
 		this.contentName = contentName;
+		this.whileRule = whileRule;
 	}
 
 	public clone(): StackElement {
-		return new StackElement(this.ruleId, this.enterPos, this.endRule, this.scopeName, this.contentName);
+		return new StackElement(this.ruleId, this.enterPos, this.endRule, this.scopeName, this.contentName, this.whileRule);
 	}
 
 	public matches(scopeName: string) : boolean {

src/rule.ts

@@ -508,6 +508,77 @@ export class BeginEndRule extends Rule {
 	}
 }
 
+export class BeginWhileRule extends Rule {
+	private _begin: RegExpSource;
+	public beginCaptures: CaptureRule[];
+	private _while: RegExpSource;
+	public whileHasBackReferences: boolean;
+	public hasMissingPatterns: boolean;
+	public patterns: number[];
+	private _cachedCompiledPatterns: RegExpSourceList;
+	private _cachedCompiledWhilePatterns: RegExpSourceList;
+
+	constructor(id: number, name: string, contentName: string, begin: string, beginCaptures: CaptureRule[], _while: string, patterns: ICompilePatternsResult) {
+		super(id, name, contentName);
+		this._begin = new RegExpSource(begin, this.id);
+		this.beginCaptures = beginCaptures;
+		this._while = new RegExpSource(_while, -2);
+		this.whileHasBackReferences = this._while.hasBackReferences;
+		this.patterns = patterns.patterns;
+		this.hasMissingPatterns = patterns.hasMissingPatterns;
+		this._cachedCompiledPatterns = null;
+		this._cachedCompiledWhilePatterns = null;
+	}
+
+	public getWhileWithResolvedBackReferences(lineText: string, captureIndices: IOnigCaptureIndex[]): string {
+		return this._while.resolveBackReferences(lineText, captureIndices);
+	}
+
+	public collectPatternsRecursive(grammar: IRuleRegistry, out: RegExpSourceList, isFirst: boolean) {
+		if (isFirst) {
+			let i: number,
+				len: number,
+				rule: Rule;
+
+			for (i = 0, len = this.patterns.length; i < len; i++) {
+				rule = grammar.getRule(this.patterns[i]);
+				rule.collectPatternsRecursive(grammar, out, false);
+			}
+		} else {
+			out.push(this._begin);
+		}
+	}
+
+	public compile(grammar: IRuleRegistry, endRegexSource: string, allowA: boolean, allowG: boolean): ICompiledRule {
+		this._precompile(grammar);
+		return this._cachedCompiledPatterns.compile(grammar, allowA, allowG);
+	}
+
+	private _precompile(grammar: IRuleRegistry): void {
+		if (!this._cachedCompiledPatterns) {
+			this._cachedCompiledPatterns = new RegExpSourceList();
+			this.collectPatternsRecursive(grammar, this._cachedCompiledPatterns, true);
+		}
+	}
+
+
+	public compileWhile(grammar: IRuleRegistry, endRegexSource: string, allowA: boolean, allowG: boolean): ICompiledRule {
+		this._precompileWhile(grammar);
+		if (this._while.hasBackReferences) {
+			this._cachedCompiledWhilePatterns.setSource(0, endRegexSource);
+		}
+		return this._cachedCompiledWhilePatterns.compile(grammar, allowA, allowG);
+	}
+
+
+	private _precompileWhile(grammar: IRuleRegistry): void {
+		if (!this._cachedCompiledWhilePatterns) {
+			this._cachedCompiledWhilePatterns = new RegExpSourceList();
+			this._cachedCompiledWhilePatterns.push(this._while.hasBackReferences ? this._while.clone() : this._while);
+		}
+	}
+}
+
 export class RuleFactory {
 
 	public static createCaptureRule(helper: IRuleFactoryHelper, name:string, contentName:string, retokenizeCapturedWithRuleId:number): CaptureRule {
@@ -542,6 +613,17 @@ export class RuleFactory {
 					);
 				}
 
+				if (desc.while) {
+					return new BeginWhileRule(
+						desc.id,
+						desc.name,
+						desc.contentName,
+						desc.begin, RuleFactory._compileCaptures(desc.beginCaptures || desc.captures, helper, repository),
+						desc.while,
+						RuleFactory._compilePatterns(desc.patterns, helper, repository)
+					);
+				}
+
 				return new BeginEndRule(
 					desc.id,
 					desc.name,
@@ -658,11 +740,7 @@ export class RuleFactory {
 
 					skipRule = false;
 
-					if (rule instanceof IncludeOnlyRule) {
-						if (rule.hasMissingPatterns && rule.patterns.length === 0) {
-							skipRule = true;
-						}
-					} else if (rule instanceof BeginEndRule) {
+					if (rule instanceof IncludeOnlyRule || rule instanceof BeginEndRule || rule instanceof BeginWhileRule) {
 						if (rule.hasMissingPatterns && rule.patterns.length === 0) {
 							skipRule = true;
 						}

src/types.ts

@@ -36,6 +36,7 @@ export interface IRawRule {
 	beginCaptures?: IRawCaptures;
 	end?:string;
 	endCaptures?: IRawCaptures;
+	while?:string;
 	patterns?: IRawRule[];
 
 	repository?: IRawRepository;

src/typings/main.d.ts

@@ -73,6 +73,7 @@ export interface StackElement {
 	ruleId: number;
 	enterPos: number;
 	endRule: string;
+	whileRule: string;
 	scopeName: string;
 	contentName: string;
 

test-cases/suite1/tests.json

@@ -77,6 +77,18 @@
 		"grammarPath": "fixtures/markdown.plist",
 		"desc": "Nested repositories in Markdown",
 		"lines": [
+			{
+				"line": "This is a paragraph",
+                "tokens": [
+                    {
+                        "value": "This is a paragraph",
+                        "scopes": [
+                            "text.html.markdown",
+                            "meta.paragraph.markdown"
+                        ]
+                    }
+                ]
+            },
 			{
 				"line": "## This is *great* stuff",
 				"tokens": [