Improve Behavior of While to More Closely Match TextMate
Issue #25 Updates while rule conditions to have be run once for each new line, from bottom of the stack to the top of the stack. This matches what I believe text mate does. Closes #25
This commit is contained in:
Matt Bierner
Родитель
749f45e5b2
Коммит
275ed31790
|
|
@ -390,34 +390,6 @@ interface IMatchResult {
|
|||
|
||||
function matchRule(grammar: Grammar, lineText: OnigString, isFirstLine: boolean, linePos: number, stack: StackElement, anchorPosition:number): IMatchResult {
|
||||
let rule = stack.getRule(grammar);
|
||||
|
||||
// Check while rule only on new lines after the line containing the begin clause
|
||||
if (rule instanceof BeginWhileRule && stack.getEnterPos() === -1 && linePos === 0) {
|
||||
|
||||
let ruleScanner = rule.compileWhile(grammar, stack.getEndRule(), isFirstLine, linePos === anchorPosition);
|
||||
let r = ruleScanner.scanner._findNextMatchSync(lineText, linePos);
|
||||
if (IN_DEBUG_MODE) {
|
||||
console.log(' scanning for while rule');
|
||||
console.log(debugCompiledRuleToString(ruleScanner));
|
||||
}
|
||||
|
||||
let doNotContinue: IMatchResult = {
|
||||
captureIndices: null,
|
||||
matchedRuleId: -3
|
||||
};
|
||||
|
||||
if (r) {
|
||||
let matchedRuleId = ruleScanner.rules[r.index];
|
||||
if (matchedRuleId != -2) {
|
||||
// we shouldn't end up here
|
||||
return doNotContinue;
|
||||
}
|
||||
} else {
|
||||
return doNotContinue;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
let ruleScanner = rule.compile(grammar, stack.getEndRule(), isFirstLine, linePos === anchorPosition);
|
||||
let r = ruleScanner.scanner._findNextMatchSync(lineText, linePos);
|
||||
if (IN_DEBUG_MODE) {
|
||||
|
|
@ -464,16 +436,73 @@ function matchRuleOrInjections(grammar: Grammar, lineText: OnigString, isFirstLi
|
|||
// injection won!
|
||||
return injectionResult;
|
||||
}
|
||||
|
||||
return matchResult;
|
||||
}
|
||||
|
||||
interface IWhileStack {
|
||||
stack: StackElement;
|
||||
rule: BeginWhileRule;
|
||||
}
|
||||
|
||||
interface IWhileCheckResult {
|
||||
stack: StackElement;
|
||||
linePos: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Walk the stack from bottom to top, and check each while condition in this order.
|
||||
* If any fails, cut off the entire stack above the failed while condition. While conditions
|
||||
* may also advance the linePosition.
|
||||
*/
|
||||
function _checkWhileConditions(grammar: Grammar, lineText: OnigString, isFirstLine: boolean, linePos: number, stack: StackElement, lineTokens: LineTokens): IWhileCheckResult {
|
||||
let whileRules: IWhileStack[] = [];
|
||||
for (let node = stack; node; node = node.pop()) {
|
||||
let nodeRule = node.getRule(grammar);
|
||||
if (nodeRule instanceof BeginWhileRule) {
|
||||
whileRules.push({
|
||||
rule: nodeRule,
|
||||
stack: node
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
for (let whileRule = whileRules.pop(); whileRule; whileRule = whileRules.pop()) {
|
||||
let ruleScanner = whileRule.rule.compileWhile(grammar, whileRule.stack.getEndRule(), isFirstLine, false);
|
||||
let r = ruleScanner.scanner._findNextMatchSync(lineText, linePos);
|
||||
if (IN_DEBUG_MODE) {
|
||||
console.log(' scanning for while rule');
|
||||
console.log(debugCompiledRuleToString(ruleScanner));
|
||||
}
|
||||
|
||||
if (r) {
|
||||
let matchedRuleId = ruleScanner.rules[r.index];
|
||||
if (matchedRuleId != -2) {
|
||||
// we shouldn't end up here
|
||||
stack = whileRule.stack.pop();
|
||||
break;
|
||||
}
|
||||
if (r.captureIndices && r.captureIndices.length) {
|
||||
linePos = r.captureIndices[0].end;
|
||||
}
|
||||
} else {
|
||||
stack = whileRule.stack.pop();
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
return { stack: stack, linePos: linePos };
|
||||
}
|
||||
|
||||
function _tokenizeString(grammar: Grammar, lineText: OnigString, isFirstLine: boolean, linePos: number, stack: StackElement, lineTokens: LineTokens): StackElement {
|
||||
const lineLength = getString(lineText).length;
|
||||
|
||||
let anchorPosition = -1;
|
||||
let STOP = false;
|
||||
|
||||
let whileCheckResult = _checkWhileConditions(grammar, lineText, isFirstLine, linePos, stack, lineTokens);
|
||||
stack = whileCheckResult.stack;
|
||||
linePos = whileCheckResult.linePos;
|
||||
|
||||
while (!STOP) {
|
||||
scanNext(); // potentially modifies linePos && anchorPosition
|
||||
}
|
||||
|
|
@ -529,16 +558,6 @@ function _tokenizeString(grammar: Grammar, lineText: OnigString, isFirstLine: bo
|
|||
STOP = true;
|
||||
return;
|
||||
}
|
||||
} else if (matchedRuleId === -3) {
|
||||
|
||||
if (IN_DEBUG_MODE) {
|
||||
console.log(' popping because a while clause no longer matches.');
|
||||
}
|
||||
|
||||
// A while clause failed
|
||||
stack = stack.pop();
|
||||
return;
|
||||
|
||||
} else {
|
||||
// We matched a rule!
|
||||
let _rule = grammar.getRule(matchedRuleId);
|
||||
|
|
|
|||
|
|
@ -100,6 +100,7 @@ describe('Tokenization /first-mate/', () => {
|
|||
|
||||
describe('Tokenization /suite1/', () => {
|
||||
assertTokenizationSuite(path.join(REPO_ROOT, 'test-cases/suite1/tests.json'));
|
||||
assertTokenizationSuite(path.join(REPO_ROOT, 'test-cases/suite1/whileTests.json'));
|
||||
});
|
||||
|
||||
describe('Matcher', () => {
|
||||
|
|
|
|||
|
|
@ -12,11 +12,11 @@
|
|||
<array>
|
||||
<dict>
|
||||
<key>include</key>
|
||||
<string>#list</string>
|
||||
<string>#alist</string>
|
||||
</dict>
|
||||
<dict>
|
||||
<key>include</key>
|
||||
<string>#number</string>
|
||||
<string>#blist</string>
|
||||
</dict>
|
||||
</array>
|
||||
<key>repository</key>
|
||||
|
|
@ -28,25 +28,60 @@
|
|||
<key>name</key>
|
||||
<string>number</string>
|
||||
</dict>
|
||||
<key>list</key>
|
||||
<key>letter</key>
|
||||
<dict>
|
||||
<key>match</key>
|
||||
<string>x</string>
|
||||
<key>name</key>
|
||||
<string>letter</string>
|
||||
</dict>
|
||||
<key>alist</key>
|
||||
<dict>
|
||||
<key>begin</key>
|
||||
<string>(^)(\L)</string>
|
||||
<string>A</string>
|
||||
<key>patterns</key>
|
||||
<array>
|
||||
<dict>
|
||||
<key>include</key>
|
||||
<string>#number</string>
|
||||
<string>#alist</string>
|
||||
</dict>
|
||||
<dict>
|
||||
<key>include</key>
|
||||
<string>#list</string>
|
||||
<string>#blist</string>
|
||||
</dict>
|
||||
<dict>
|
||||
<key>include</key>
|
||||
<string>#letter</string>
|
||||
</dict>
|
||||
</array>
|
||||
<key>name</key>
|
||||
<string>list</string>
|
||||
<string>alist</string>
|
||||
<key>while</key>
|
||||
<string>(^|\G)[ ]{2}</string>
|
||||
<string>a</string>
|
||||
</dict>
|
||||
<key>blist</key>
|
||||
<dict>
|
||||
<key>begin</key>
|
||||
<string>B</string>
|
||||
<key>patterns</key>
|
||||
<array>
|
||||
<dict>
|
||||
<key>include</key>
|
||||
<string>#alist</string>
|
||||
</dict>
|
||||
<dict>
|
||||
<key>include</key>
|
||||
<string>#blist</string>
|
||||
</dict>
|
||||
<dict>
|
||||
<key>include</key>
|
||||
<string>#number</string>
|
||||
</dict>
|
||||
</array>
|
||||
<key>name</key>
|
||||
<string>blist</string>
|
||||
<key>while</key>
|
||||
<string>b</string>
|
||||
</dict>
|
||||
</dict>
|
||||
<key>scopeName</key>
|
||||
|
|
|
|||
|
|
@ -1015,81 +1015,5 @@
|
|||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"grammars": [
|
||||
"fixtures/whileLang.plist"
|
||||
],
|
||||
"grammarPath": "fixtures/whileLang.plist",
|
||||
"desc": "Tests that matching of while only tests on first line",
|
||||
"lines": [
|
||||
{
|
||||
"line": "L 1",
|
||||
"tokens": [
|
||||
{
|
||||
"scopes": [
|
||||
"text.whileLang",
|
||||
"list"
|
||||
],
|
||||
"value": "L"
|
||||
},
|
||||
{
|
||||
"scopes": [
|
||||
"text.whileLang",
|
||||
"list"
|
||||
],
|
||||
"value": " "
|
||||
},
|
||||
{
|
||||
"scopes": [
|
||||
"text.whileLang",
|
||||
"list",
|
||||
"number"
|
||||
],
|
||||
"value": "1"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"line": " 23",
|
||||
"tokens": [
|
||||
{
|
||||
"scopes": [
|
||||
"text.whileLang",
|
||||
"list"
|
||||
],
|
||||
"value": " "
|
||||
},
|
||||
{
|
||||
"scopes": [
|
||||
"text.whileLang",
|
||||
"list",
|
||||
"number"
|
||||
],
|
||||
"value": "2"
|
||||
},
|
||||
{
|
||||
"scopes": [
|
||||
"text.whileLang",
|
||||
"list",
|
||||
"number"
|
||||
],
|
||||
"value": "3"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"line": "4",
|
||||
"tokens": [
|
||||
{
|
||||
"scopes": [
|
||||
"text.whileLang",
|
||||
"number"
|
||||
],
|
||||
"value": "4"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
|
|
|
|||
|
|
@ -0,0 +1,395 @@
|
|||
[
|
||||
{
|
||||
"grammars": [
|
||||
"fixtures/whileLang.plist"
|
||||
],
|
||||
"grammarPath": "fixtures/whileLang.plist",
|
||||
"desc": "While should match begin and stop on next line if while condition fails",
|
||||
"lines": [
|
||||
{
|
||||
"line": "A x",
|
||||
"tokens": [
|
||||
{
|
||||
"scopes": [
|
||||
"text.whileLang",
|
||||
"alist"
|
||||
],
|
||||
"value": "A"
|
||||
},
|
||||
{
|
||||
"scopes": [
|
||||
"text.whileLang",
|
||||
"alist"
|
||||
],
|
||||
"value": " "
|
||||
},
|
||||
{
|
||||
"scopes": [
|
||||
"text.whileLang",
|
||||
"alist",
|
||||
"letter"
|
||||
],
|
||||
"value": "x"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"line": "c",
|
||||
"tokens": [
|
||||
{
|
||||
"scopes": [
|
||||
"text.whileLang"
|
||||
],
|
||||
"value": "c"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"grammars": [
|
||||
"fixtures/whileLang.plist"
|
||||
],
|
||||
"grammarPath": "fixtures/whileLang.plist",
|
||||
"desc": "While should match multiple lines while condition holds",
|
||||
"lines": [
|
||||
{
|
||||
"line": "A x",
|
||||
"tokens": [
|
||||
{
|
||||
"scopes": [
|
||||
"text.whileLang",
|
||||
"alist"
|
||||
],
|
||||
"value": "A"
|
||||
},
|
||||
{
|
||||
"scopes": [
|
||||
"text.whileLang",
|
||||
"alist"
|
||||
],
|
||||
"value": " "
|
||||
},
|
||||
{
|
||||
"scopes": [
|
||||
"text.whileLang",
|
||||
"alist",
|
||||
"letter"
|
||||
],
|
||||
"value": "x"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"line": "ax x",
|
||||
"tokens": [
|
||||
{
|
||||
"scopes": [
|
||||
"text.whileLang",
|
||||
"alist"
|
||||
],
|
||||
"value": "a"
|
||||
},
|
||||
{
|
||||
"scopes": [
|
||||
"text.whileLang",
|
||||
"alist",
|
||||
"letter"
|
||||
],
|
||||
"value": "x"
|
||||
},
|
||||
{
|
||||
"scopes": [
|
||||
"text.whileLang",
|
||||
"alist"
|
||||
],
|
||||
"value": " "
|
||||
},
|
||||
{
|
||||
"scopes": [
|
||||
"text.whileLang",
|
||||
"alist",
|
||||
"letter"
|
||||
],
|
||||
"value": "x"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"line": "c",
|
||||
"tokens": [
|
||||
{
|
||||
"scopes": [
|
||||
"text.whileLang"
|
||||
],
|
||||
"value": "c"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"grammars": [
|
||||
"fixtures/whileLang.plist"
|
||||
],
|
||||
"grammarPath": "fixtures/whileLang.plist",
|
||||
"desc": "While condition can match anywhere in line",
|
||||
"lines": [
|
||||
{
|
||||
"line": "A x",
|
||||
"tokens": [
|
||||
{
|
||||
"scopes": [
|
||||
"text.whileLang",
|
||||
"alist"
|
||||
],
|
||||
"value": "A"
|
||||
},
|
||||
{
|
||||
"scopes": [
|
||||
"text.whileLang",
|
||||
"alist"
|
||||
],
|
||||
"value": " "
|
||||
},
|
||||
{
|
||||
"scopes": [
|
||||
"text.whileLang",
|
||||
"alist",
|
||||
"letter"
|
||||
],
|
||||
"value": "x"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"line": "xax",
|
||||
"tokens": [
|
||||
{
|
||||
"scopes": [
|
||||
"text.whileLang",
|
||||
"alist"
|
||||
],
|
||||
"value": "xa"
|
||||
},
|
||||
{
|
||||
"scopes": [
|
||||
"text.whileLang",
|
||||
"alist",
|
||||
"letter"
|
||||
],
|
||||
"value": "x"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"grammars": [
|
||||
"fixtures/whileLang.plist"
|
||||
],
|
||||
"grammarPath": "fixtures/whileLang.plist",
|
||||
"desc": "Begin of while should consume entire rest of line.",
|
||||
"lines": [
|
||||
{
|
||||
"line": "A x B 1",
|
||||
"tokens": [
|
||||
{
|
||||
"scopes": [
|
||||
"text.whileLang",
|
||||
"alist"
|
||||
],
|
||||
"value": "A"
|
||||
},
|
||||
{
|
||||
"scopes": [
|
||||
"text.whileLang",
|
||||
"alist"
|
||||
],
|
||||
"value": " "
|
||||
},
|
||||
{
|
||||
"scopes": [
|
||||
"text.whileLang",
|
||||
"alist",
|
||||
"letter"
|
||||
],
|
||||
"value": "x"
|
||||
},
|
||||
{
|
||||
"scopes": [
|
||||
"text.whileLang",
|
||||
"alist"
|
||||
],
|
||||
"value": " "
|
||||
},
|
||||
{
|
||||
"scopes": [
|
||||
"text.whileLang",
|
||||
"alist",
|
||||
"blist"
|
||||
],
|
||||
"value": "B"
|
||||
},
|
||||
{
|
||||
"scopes": [
|
||||
"text.whileLang",
|
||||
"alist",
|
||||
"blist"
|
||||
],
|
||||
"value": " "
|
||||
},
|
||||
{
|
||||
"scopes": [
|
||||
"text.whileLang",
|
||||
"alist",
|
||||
"blist",
|
||||
"number"
|
||||
],
|
||||
"value": "1"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"grammars": [
|
||||
"fixtures/whileLang.plist"
|
||||
],
|
||||
"grammarPath": "fixtures/whileLang.plist",
|
||||
"desc": "Nested whiles should match using only inner most while on a mached line",
|
||||
"lines": [
|
||||
{
|
||||
"line": "AB",
|
||||
"tokens": [
|
||||
{
|
||||
"scopes": [
|
||||
"text.whileLang",
|
||||
"alist"
|
||||
],
|
||||
"value": "A"
|
||||
},
|
||||
{
|
||||
"scopes": [
|
||||
"text.whileLang",
|
||||
"alist",
|
||||
"blist"
|
||||
],
|
||||
"value": "B"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"line": "abx1",
|
||||
"tokens": [
|
||||
{
|
||||
"scopes": [
|
||||
"text.whileLang",
|
||||
"alist",
|
||||
"blist"
|
||||
],
|
||||
"value": "abx"
|
||||
},
|
||||
{
|
||||
"scopes": [
|
||||
"text.whileLang",
|
||||
"alist",
|
||||
"blist",
|
||||
"number"
|
||||
],
|
||||
"value": "1"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"grammars": [
|
||||
"fixtures/whileLang.plist"
|
||||
],
|
||||
"grammarPath": "fixtures/whileLang.plist",
|
||||
"desc": "Nested whiles should check line for outer most while to inner most while",
|
||||
"lines": [
|
||||
{
|
||||
"line": "AB",
|
||||
"tokens": [
|
||||
{
|
||||
"scopes": [
|
||||
"text.whileLang",
|
||||
"alist"
|
||||
],
|
||||
"value": "A"
|
||||
},
|
||||
{
|
||||
"scopes": [
|
||||
"text.whileLang",
|
||||
"alist",
|
||||
"blist"
|
||||
],
|
||||
"value": "B"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"line": "b1",
|
||||
"tokens": [
|
||||
{
|
||||
"scopes": [
|
||||
"text.whileLang"
|
||||
],
|
||||
"value": "b1"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"grammars": [
|
||||
"fixtures/whileLang.plist"
|
||||
],
|
||||
"grammarPath": "fixtures/whileLang.plist",
|
||||
"desc": "Nested whiles should move line ahead before checking other conditions",
|
||||
"lines": [
|
||||
{
|
||||
"line": "AB",
|
||||
"tokens": [
|
||||
{
|
||||
"scopes": [
|
||||
"text.whileLang",
|
||||
"alist"
|
||||
],
|
||||
"value": "A"
|
||||
},
|
||||
{
|
||||
"scopes": [
|
||||
"text.whileLang",
|
||||
"alist",
|
||||
"blist"
|
||||
],
|
||||
"value": "B"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"line": "bax",
|
||||
"tokens": [
|
||||
{
|
||||
"scopes": [
|
||||
"text.whileLang",
|
||||
"alist"
|
||||
],
|
||||
"value": "ba"
|
||||
},
|
||||
{
|
||||
"scopes": [
|
||||
"text.whileLang",
|
||||
"alist",
|
||||
"letter"
|
||||
],
|
||||
"value": "x"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
Загрузка…
Ссылка в новой задаче