microsoft
/
gather
зеркало из https://github.com/microsoft/gather.git
1
0
Форкнуть 0
Код Задачи Пакеты Проекты Релизы Вики Активность

Bugfix: Update parser to work in lab, and recover from errors

This commit is contained in:
Andrew Head 2018-12-14 14:10:13 -08:00
Родитель 134130aaab
Коммит 23de7f4f92
5 изменённых файлов: 60 добавлений и 63 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

6
package.json
Просмотреть файл

@ -9,18 +9,18 @@
"jupyterlab"
],
"scripts": {
"build_parser": "node ./node_modules/jison/lib/cli.js --outfile lib/parsers/python/python3.js.txt src/parsers/python/python3.jison",
"build_parser": "node ./node_modules/jison/lib/cli.js --outfile src/parsers/python/python3.js src/parsers/python/python3.jison",
"build": "tsc",
"build_nb_extension": "npm run build && npx webpack",
"watch_nb_extension": "npx webpack -w",
"install_nb_extension": "jupyter nbextension install dist && jupyter nbextension enable dist/gather",
"prepack": "npm run clean && npm run build && npm run build_parser",
"prepack": "npm run clean && npm run build_parser && npm run build",
"clean": "rimraf lib",
"watch": "tsc -w",
"test": "mocha -r ts-node/register src/test/*.test.ts"
},
"files": [
"lib/**/*.{d.ts,txt,eot,gif,html,jpg,js,js.map,json,png,svg,woff2,ttf}",
"lib/**/*.{d.ts,eot,gif,html,jpg,js,js.map,json,png,svg,woff2,ttf}",
"style/**/*.{css,eot,gif,html,jpg,json,png,svg,woff2,ttf}"
],
"jupyterlab": {

80
src/parsers/python/python3.jison
Просмотреть файл

@ -3,26 +3,6 @@
/* https://docs.python.org/3.4/reference/grammar.html */
/* lexical gammar */
%{
var indents = [0],
indent = 0,
dedents = 0,
loc = undefined,
partial = undefined,
// we don't need to implement a full stack here to ensure symmetry
// because it's ensured by the grammar
brackets_count = 0;
var keywords = [
"continue", "nonlocal", "finally", "lambda", "return", "assert",
"global", "import", "except", "raise", "break", "False", "class",
"while", "yield", "None", "True", "from", "with", "elif", "else",
"pass", "for", "try", "def", "and", "del", "not", "is", "as", "if",
"or", "in"
]
%}
%lex
uppercase [A-Z]
@ -86,36 +66,39 @@ imagnumber ({floatnumber}|{intpart})[jJ]
<INITIAL,INLINE><<EOF>> %{
// if the last statement in indented, need to force a dedent before EOF
if (indents.length > 1) {
if (this.indents == undefined) this.indents == [0];
if (this.indents.length > 1) {
this.begin( 'DEDENTS' );
this.unput(' '); // make sure EOF is not triggered
dedents = 1;
indents.pop();
this.dedents = 1;
this.indents.pop();
} else {
return 'EOF';
}
%}
<INITIAL>\ %{ indent += 1 %}
<INITIAL>\t %{ indent = ( indent + 8 ) & -7 %}
<INITIAL>\n %{ indent = 0 %} // blank line
<INITIAL>\ %{ if (this.indent == undefined) this.indent = 0; this.indent += 1 %}
<INITIAL>\t %{ if (this.indent == undefined) this.indent = 0; this.indent = ( this.indent + 8 ) & -7 %}
<INITIAL>\n %{ this.indent = 0 %} // blank line
<INITIAL>\#[^\n]* /* skip comments */
<INITIAL>. %{
this.unput( yytext )
var last = indents[ indents.length - 1 ]
if ( indent > last ) {
if (this.indents == undefined) this.indents = [0];
var last = this.indents[ this.indents.length - 1 ]
if (this.indent == undefined) this.indent = 0;
if ( this.indent > last ) {
this.begin( 'INLINE' )
indents.push( indent )
this.indents.push( this.indent )
return 'INDENT'
} else if ( indent < last ) {
} else if ( this.indent < last ) {
this.begin( 'DEDENTS' )
dedents = 0 // how many dedents occured
while( indents.length ) {
dedents += 1
indents.pop()
last = indents[ indents.length - 1 ]
if ( last == indent ) break
this.dedents = 0 // how many dedents occured
while( this.indents.length ) {
this.dedents += 1
this.indents.pop()
last = this.indents[ this.indents.length - 1 ]
if ( last == this.indent ) break
}
if ( !indents.length ) {
if ( !this.indents.length ) {
throw new Error( "TabError: Inconsistent" )
}
} else {
@ -124,7 +107,8 @@ imagnumber ({floatnumber}|{intpart})[jJ]
%}
<DEDENTS>. %{
this.unput( yytext )
if ( dedents-- > 0 ) {
if (this.dedents == undefined) this.dedents = 0;
if ( this.dedents-- > 0 ) {
return 'DEDENT'
} else {
this.begin( 'INLINE' )
@ -133,8 +117,9 @@ imagnumber ({floatnumber}|{intpart})[jJ]
<INLINE>\n %{
// implicit line joining
if ( brackets_count <= 0 ) {
indent = 0;
if (this.brackets_count == undefined) this.brackets_count = 0;
if ( this.brackets_count <= 0 ) {
this.indent = 0;
this.begin( 'INITIAL' )
return 'NEWLINE'
}
@ -155,10 +140,11 @@ imagnumber ({floatnumber}|{intpart})[jJ]
%}
<INLINE>{integer} return 'NUMBER'
<INLINE>{operators} %{
if (this.brackets_count == undefined) this.brackets_count = 0;
if ( yytext == '{' || yytext == '[' || yytext == '(' ) {
brackets_count += 1
this.brackets_count += 1
} else if ( yytext == '}' || yytext == ']' || yytext == ')' ) {
brackets_count -= 1
this.brackets_count -= 1
}
return yytext
%}
@ -185,7 +171,14 @@ imagnumber ({floatnumber}|{intpart})[jJ]
return 'BYTES'
%}
<INLINE>{identifier} %{
return ( keywords.indexOf( yytext ) == -1 )
this.keywords = [
"continue", "nonlocal", "finally", "lambda", "return", "assert",
"global", "import", "except", "raise", "break", "False", "class",
"while", "yield", "None", "True", "from", "with", "elif", "else",
"pass", "for", "try", "def", "and", "del", "not", "is", "as", "if",
"or", "in"
]
return ( this.keywords.indexOf( yytext ) == -1 )
? 'NAME'
: yytext;
%}
@ -1299,4 +1292,3 @@ yield_arg
| testlist
;

30
src/parsers/python/python_parser.ts
Просмотреть файл

@ -1,6 +1,17 @@
import * as python3txt from './python3.js.txt';
import * as fs from 'fs';
import * as path from 'path';
import { parse as python3Parse, parser } from './python3';
/**
* Reset the lexer state after an error. Otherwise, parses after a failed parse can fail too.
*/
let yy = parser.yy as any;
let oldParseError = yy.parseError;
oldParseError = function(text: String, hash: any) {
this.indents = [0];
this.indent = 0;
this.dedents = 0;
this.brackets_count = 0;
oldParseError.call(this, text, hash);
};
/**
* This is the main interface for parsing code.
@ -13,18 +24,7 @@ export function parse(program: string): IModule {
// eliminate byte order mark
program = program.slice(1);
}
// We avoid using require since loading/unloading the module causes a memory leak.
let exports = { parse: (s: string): IModule => null };
if (typeof(python3txt) == "string") {
eval(python3txt); // overwrites parse function
// During unit tests, python3txt will be an object, not a string, causing unexpected
// behavior. If python3txt isn't a string, read it using `fs`.
} else {
const fname = path.join(path.dirname(__filename), 'python3.js.txt');
const python3txtString = fs.readFileSync(fname).toString();
eval(python3txtString); // overwrites parse function
}
return exports.parse(program);
return python3Parse(program);
}

5
src/test/parser.test.ts
Просмотреть файл

@ -56,6 +56,11 @@ describe('python parser', () => {
last_column: 10
});
});
it('does not crash on correct code after parsing bad code', () => {
expect(() => parse("print(1\n")).to.throw();
expect(() => parse("a + 1\nb = a\n")).not.to.throw();
});
});
describe('ast walker', () => {