From 7a047f75b9d8e1d2d9fdaa090c9650f7cc41f0f6 Mon Sep 17 00:00:00 2001 From: Michal Moskal Date: Mon, 11 Apr 2016 19:01:47 -0700 Subject: [PATCH] Improving converter --- Jakefile | 10 ++ ast/ast.ts | 2 + ast/converter.ts | 259 ++++++++++++++++++++++++++++--------- ast/mddocs.ts | 3 +- editor/scriptProperties.ts | 2 +- noderunner/nrunner.ts | 39 +++++- tools/client.ts | 17 ++- 7 files changed, 262 insertions(+), 70 deletions(-) diff --git a/Jakefile b/Jakefile index b1e37d6c..7edfca8f 100644 --- a/Jakefile +++ b/Jakefile @@ -302,6 +302,16 @@ var concatMap = { "build/embedded", "build/editor" , ], + "build/tdast.js": [ + "build/browser", + "build/rt", + "build/ast", + "build/storage", + "build/embedded", + "build/api.js", + "generated/langs.js", + "build/libraries.js", + ], }; // Just a dumb concatenation diff --git a/ast/ast.ts b/ast/ast.ts index e3dc4c15..0ff5b37c 100644 --- a/ast/ast.ts +++ b/ast/ast.ts @@ -48,6 +48,7 @@ module TDev.AST { public _error:string = null; public isInvisible:boolean; public annotations:RT.AstAnnotation[]; + public _converterValue:string; private isAstNode() { return true; } public isPlaceholder() { return false; } @@ -2279,6 +2280,7 @@ module TDev.AST { public _isCaptured:boolean; // set by TypeChecker for locals captured in closures public _lastWriteLocation:Stmt; public _converterAction:InlineAction; + public _converterUses:number; public isByRef() { return this._isMutable && this._isCaptured } public writeWithType(app:App, tw:TokenWriter) diff --git a/ast/converter.ts b/ast/converter.ts index fc639039..8e0d89ac 100644 --- a/ast/converter.ts +++ b/ast/converter.ts @@ -153,6 +153,7 @@ module TDev.AST { visitAstNode(n:AstNode) { + delete n._converterValue this.visitChildren(n); } @@ -177,6 +178,106 @@ module TDev.AST { if (eh.isAwait) this.numAwaits++ } + + visitThingRef(t:ThingRef) { + var l = t.referencedLocal() + if (l && typeof l._converterUses == "number") + l._converterUses++ + } + + visitAction(a:Action) { + a.getOutParameters().forEach(op => { + op.local._converterUses = 0 + }) + super.visitAction(a) + } + } + + export interface Td2TsOptions { + text?: string; + useExtensions?: boolean; + apiInfo?: ApisInfo; + } + + export interface ParameterDesc { + name: string; + description: string; + type: string; + initializer?: string; + defaults?: string[]; + } + + export enum SymbolKind { + None, + Method, + Property, + Function, + Variable, + Module, + Enum, + EnumMember + } + + export interface CommentAttrs { + shim?: string; + enumval?: string; + helper?: string; + help?: string; + async?: boolean; + block?: string; + blockId?: string; + blockGap?: string; + blockExternalInputs?: boolean; + blockStatement?: boolean; + blockImportId?: string; + color?: string; + icon?: string; + imageLiteral?: number; + weight?: number; + + // on interfaces + indexerGet?: string; + indexerSet?: string; + + _name?: string; + jsDoc?: string; + paramHelp?: StringMap; + // foo.defl=12 -> paramDefl: { foo: "12" } + paramDefl: StringMap; + } + + + export interface SymbolInfo { + attributes: CommentAttrs; + name: string; + namespace: string; + kind: SymbolKind; + parameters: ParameterDesc[]; + retType: string; + isContextual?: boolean; + } + + export interface ApisInfo { + byQName: StringMap; + } + + + export function td2ts(options:Td2TsOptions) { + AST.reset(); + AST.loadScriptAsync((s) => Promise.as(s == "" ? options.text : null)); + var r = new TDev.AST.Converter(Script, options).run() + return r; + } + + export function testConverterAsync() { + return Util.httpGetJsonAsync("http://localhost:4242/editor/local/apiinfo.json") + .then(json => { + var r = new TDev.AST.Converter(Script, { + useExtensions: true, + apiInfo: json + }).run() + return r.text; + }) } export class Converter @@ -187,9 +288,8 @@ module TDev.AST { private currAsync:Call; private apis:StringMap = {}; private allEnums:StringMap = {}; - public useExtensions = false; - constructor(private app:App) + constructor(private app:App, private options:Td2TsOptions = {}) { super() } @@ -202,8 +302,9 @@ module TDev.AST { keys.sort((a, b) => this.apis[a] - this.apis[b]) var newApis = {} keys.forEach(k => newApis[k] = this.apis[k]) + var txt = (this.tw.finalize().trim() + "\n").replace(/\n+/g, "\n") return { - text: this.tw.finalize(), + text: txt, apis: newApis, } } @@ -235,8 +336,8 @@ module TDev.AST { "Boolean": "boolean", "Nothing": "void", "DateTime": "Date", - "Json Object": "JsonObject", - "Json Builder": "JsonBuilder", + "Json Object": "{}", + "Json Builder": "{}", "Buffer": "Buffer", } @@ -280,6 +381,11 @@ module TDev.AST { return this.type(l.getKind()) } + private globalName(n:string) + { + return this.tw.globalCtx.quote(n, 0) + } + visitAstNode(n:AstNode) { this.visitChildren(n); @@ -321,6 +427,8 @@ module TDev.AST { return 5 // '!= ""' return 50 } + } else if (p.getName() == "mod" && p.parentKind.getName() == "Math") { + return 20 } else if (e instanceof Call && e.awaits()) { return 40 } @@ -350,52 +458,40 @@ module TDev.AST { } static prefixGlue:StringMap = { - "String->replace": "td.replaceAll", - "String->starts with": "td.startsWith", - "App->server setting": "td.serverSetting", - "App->log": "td.log", + "App->log": "console.log", "Time->now": "new Date", "Json Builder->keys": "Object.keys", "Json Object->keys": "Object.keys", "String Map->keys": "Object.keys", "Contract->assert": "assert", "Bits->string to buffer": "new Buffer", - "Time->sleep": "td.sleepAsync", - "String->to number": "parseFloat", - "Json Builder->copy from": "td.jsonCopyFrom", - "String->contains": "td.stringContains", - "Collection->to json": "td.arrayToJson", - "Collection->ordered by": "td.orderedBy", + "Time->sleep": "basic.sleep", + "String->to number": "parseInt", "Web->decode url": "decodeURIComponent", "Web->decode uri component": "decodeURIComponent", "Web->encode uri component": "encodeURIComponent", "Web->encode url": "encodeURIComponent", - "Json Object->to collection": "asArray", - "Json Builder->to collection": "asArray", - "Math->clamp": "td.clamp", + "Math->clamp": "Math.clamp", "Math->min": "Math.min", "Math->max": "Math.max", "Math->round": "Math.round", "Math->floor": "Math.floor", - "Math->random": "td.randomInt", - "Math->random range": "td.randomRange", - "Math->random normalized": "td.randomNormalized", + "Math->random": "Math.random", + "Math->random range": "Math.randomRange", + "Math->random normalized": "Math.randomNormalized", "Json Builder->to string": "td.toString", "Json Object->to string": "td.toString", "Json Builder->to number": "td.toNumber", "Json Object->to number": "td.toNumber", - "App->create logger": "td.createLogger", - "Web->create request": "td.createRequest", - "Web->download json": "td.downloadJsonAsync", - "Web->download": "td.downloadTextAsync", - "Contract->requires": "assert", - "Buffer->sha256": "td.sha256", + "♻ micro:bit->plot image": "basic.showLeds", + "♻ micro:bit->create image": "images.createImage", } static methodRepl:StringMap = { "Buffer->concat": "concat", "Buffer->to string": "toString", "Number->to string": "toString", + "String->replace": "replaceAll", "String->split": "split", "String->substring": "substr", "String->to upper case": "toUpperCase", @@ -421,6 +517,55 @@ module TDev.AST { }) } + fixupArgs(e:Call, nameOverride:string) { + if (!e.args[0]) + return + + var th = e.args[0].getThing() + + if (!(th instanceof SingletonDef)) + return + + if (th.getKind() instanceof ThingSetKind) + return + + if (!this.options.apiInfo) + return + + var p = e.getCalledProperty() + var fullName = nameOverride || this.globalName(e.args[0].getThing().getName()) + "." + this.globalName(p.getName()) + + var apiInfo = this.options.apiInfo.byQName[fullName] + if (!apiInfo) { + this.tw.write("/*WARN: no api " + fullName + "*/") + return + } + + e.args.slice(1).forEach((a, i) => { + var litVal:string = a.getLiteral() + if (!litVal || typeof litVal != "string") return; + litVal = litVal.toLowerCase() + var pi = apiInfo.parameters[i] + if (!pi) return + + var enumInfo = this.options.apiInfo.byQName[pi.type] + if (enumInfo && enumInfo.kind == SymbolKind.Enum) { + var members = Util.values(this.options.apiInfo.byQName) + .filter(e => e.namespace == pi.type && + (e.name.toLowerCase() == litVal || + (e.attributes.block || "").toLowerCase() == litVal)) + if (members.length >= 1) { + a._converterValue = members[0].namespace + "." + members[0].name + if (members.length > 1) { + a._converterValue += " /*WARN: more possible!*/" + } + } else { + a._converterValue = pi.type + "." + a.getLiteral() + " /*WARN: not found*/" + } + } + }) + } + visitCallInner(e:Call) { var p = e.getCalledProperty() @@ -443,6 +588,8 @@ module TDev.AST { return } + this.fixupArgs(e, Converter.prefixGlue[pn]) + if (infixPri) { if (p.getName() == "-" && e.args[0].getLiteral() === 0.0) { this.tw.op0("-") @@ -713,6 +860,7 @@ module TDev.AST { "\u2260": "!=", "\u2264": "<=", "\u2265": ">=", + "mod": "%", } printOp(s:string) @@ -736,6 +884,11 @@ module TDev.AST { visitLiteral(l:Literal) { + if (l._converterValue) { + this.tw.write(l._converterValue) + return + } + if (l.data === undefined) return if (typeof l.data == "number") this.tw.write(l.stringForm || l.data.toString()) @@ -915,14 +1068,14 @@ module TDev.AST { isOwnExtension(a:Action) { - if (!this.useExtensions || !this.isExtension(a)) return false + if (!this.options.useExtensions || !this.isExtension(a)) return false var r = this.getFirstRecord(a) return (r && r.parent == this.app) } isExtension(a:Action) { - if (!this.useExtensions && a.parent == this.app) + if (!this.options.useExtensions && a.parent == this.app) return false if (a.isActionTypeDef()) @@ -1085,8 +1238,8 @@ module TDev.AST { this.tw.globalId(a) this.pcommaSep(a.getInParameters().slice(1), printP) } else { - if (!a.isPrivate) - this.tw.kw("export") + //if (!a.isPrivate) + // this.tw.kw("export") if (useAsync && !a.isAtomic) this.tw.kw("async") this.tw.kw("function") @@ -1131,15 +1284,21 @@ module TDev.AST { this.tw.jsid(info.optsName).op0(");").nl() } - a.getOutParameters().forEach(p => { - this.tw.kw("let") - this.localDef(p.local) - this.tw.semiNL() - }) + var hasOutP = !!a.getOutParameters().filter(p => p.local._converterUses > 0)[0] + + + if (hasOutP) + a.getOutParameters().forEach(p => { + this.tw.kw("let") + this.localDef(p.local) + this.tw.semiNL() + }) this.codeBlockInner(info.stmts) - if (a.getOutParameters().length == 1) { + if (!hasOutP) { + // nothing to do + } else if (a.getOutParameters().length == 1) { this.tw.kw("return") this.localName(a.getOutParameters()[0].local).semiNL() } else if (a.getOutParameters().length > 1) { @@ -1187,18 +1346,14 @@ module TDev.AST { visitLibraryRef(l:LibraryRef) { - var modName = JSON.stringify("./" + l.getName().replace(/\s/g, "-")) - this.tw.kw("import * as "); - this.tw.globalId(l).keyword("from").sep().write(modName).nl(); } visitRecordDef(r:RecordDef) { - this.tw.kw("export class").sep() - this.tw.globalId(r).nl() - this.tw.kw(" extends td.JsonRecord").nl().beginBlock() + this.tw.kw("class").sep() + this.tw.globalId(r).beginBlock() r.getFields().forEach(f => { - this.tw.kw("@json public").sep() + this.tw.kw("public").sep() this.simpleId(f.getName()) this.tw.op0(":").sep() this.type(f.dataKind) @@ -1208,27 +1363,11 @@ module TDev.AST { this.tw.semiNL() }) - this.tw.write("static createFromJson(o:JsonObject) { let r = new ") - this.tw.globalId(r).write("(); r.fromJson(o); return r; }").nl() - var exts = this.app.actions().filter(a => this.isOwnExtension(a) && this.getFirstRecord(a) == r) exts.forEach(e => this.visitAction(e)) this.tw.endBlock() this.tw.nl() - - this.tw.kw("export interface").sep() - this.tw.globalId(r, "I") - this.tw.beginBlock() - r.getFields().forEach(f => { - this.simpleId(f.getName()) - this.tw.op0("?:").sep() - this.type(f.dataKind) - this.tw.semiNL() - }) - this.tw.endBlock() - - this.tw.nl() } visitApp(a:App) diff --git a/ast/mddocs.ts b/ast/mddocs.ts index 7935af76..29e12f42 100644 --- a/ast/mddocs.ts +++ b/ast/mddocs.ts @@ -54,7 +54,7 @@ module TDev.AST.MdDocs { if (l.getName() == m[0]) act = act || l.getPublicActions().filter(a => a.getName() == m[1])[0] }) - if (act) { + if (act && converter) { return quoteCode(converter.renderSig(act)) } else { warn("cannot find lib action: " + arg) @@ -143,6 +143,7 @@ module TDev.AST.MdDocs { function mkSnippet(stmts:Stmt[]) { + if (!converter) return "[" + stmts.length + " stmts]" return quoteCode(converter.renderSnippet(stmts)) } diff --git a/editor/scriptProperties.ts b/editor/scriptProperties.ts index 93d297aa..e817a897 100644 --- a/editor/scriptProperties.ts +++ b/editor/scriptProperties.ts @@ -224,7 +224,7 @@ module TDev HTML.mkButton(lf("public -> test"), () => this.publicToTest()), HTML.mkButton(lf("time tc"), () => Editor.testScriptTc()), HTML.mkButton(lf("speech driven"), () => TheEditor.calculator.searchApi.listenToSpeech()), - HTML.mkButton(lf("TypeScript"), () => ModalDialog.showText(new AST.Converter(TDev.Script).run().text)), + HTML.mkButton(lf("TypeScript"), () => AST.testConverterAsync().done(ModalDialog.showText)), HTML.mkButton(lf("Markdown"), () => ModalDialog.showText(AST.MdDocs.toMD(TDev.Script))) ) : undefined, Browser.EditorSettings.changeSkillLevelDiv(this.editor, Ticks.changeSkillScriptProperties, "formLine marginBottom"), diff --git a/noderunner/nrunner.ts b/noderunner/nrunner.ts index 476ed0f2..10d05f8d 100644 --- a/noderunner/nrunner.ts +++ b/noderunner/nrunner.ts @@ -1734,15 +1734,42 @@ function mergetest(args:string[]) }) } +function td2tsOpts() { + var f = fs.readFileSync("c:/dev/temp/apiinfo.json", "utf8") + return { + text: "", + useExtensions: true, + apiInfo: JSON.parse(f) + } +} + +function tsall(files:string[]) +{ + var used = {} + var opts = td2tsOpts() + fs.readdirSync("dls").forEach(fn => { + console.log(fn) + var t = fs.readFileSync("dls/" + fn, "utf8") + opts.text = t + var r = TDev.AST.td2ts(opts) + var tt = r.text.replace(/".*?"/g, "STR").replace(/[^a-zA-Z]/g, "") + //if (used[tt]) return; + used[tt] = 1 + + fs.writeFileSync("tss/" + fn + ".ts", r.text) + // fs.writeFileSync("apis.json", JSON.stringify(r.apis, null, 2)) + //console.log("out.ts and apis.json written") + }) +} + function ts(files:string[]) { var t = fs.readFileSync(files[0], "utf8") - TDev.AST.reset(); - TDev.AST.loadScriptAsync((s) => TDev.Promise.as(s == "" ? t : null)); - var r = new TDev.AST.Converter(TDev.Script).run() + var opts = td2tsOpts() + opts.text = t + var r = TDev.AST.td2ts(opts) fs.writeFileSync("out.ts", r.text) - fs.writeFileSync("apis.json", JSON.stringify(r.apis, null, 2)) - console.log("out.ts and apis.json written") + // fs.writeFileSync("apis.json", JSON.stringify(r.apis, null, 2)) } function mddocs(files:string[]) @@ -2097,6 +2124,8 @@ export function globalInit() compilerTest(); } else if (process.argv[2] == "ts") { ts(process.argv.slice(3)) + } else if (process.argv[2] == "tsall") { + tsall(process.argv.slice(3)) } else if (process.argv[2] == "mddocs") { mddocs(process.argv.slice(3)) } else { diff --git a/tools/client.ts b/tools/client.ts index c85a8dc5..900d48eb 100644 --- a/tools/client.ts +++ b/tools/client.ts @@ -3633,6 +3633,8 @@ function dlscripttext(args:string[]) var pushMode = args[1] == "push" + var liteMode = true + var tm0 = 1420099200 var start = Date.now(); var k = tdliteKey() @@ -3645,12 +3647,21 @@ function dlscripttext(args:string[]) var oneup = () => { if (--num == 0) { var tm = ("00000" + Math.round((Date.now() - start) / 1000)).slice(-6) - console.log(tm, "DONE", files[i], i + "/" + files.length) + console.log(tm, "DONE", files[i], i + "/" + files.length, items.length) loop(i+1) } } var pref = "https://www.touchdevelop.com/api/" + var suff = "" + + if (liteMode) { + var kk = tdliteKey(); + suff = kk.key + pref = kk.liteUrl + "api/" + var now = Date.now()/1000 + items = items.filter(it => it.updateid == it.id && !it.editor && now - it.time < 9*30*24*3600) + } items.forEach(s => { ++num; @@ -3669,11 +3680,11 @@ function dlscripttext(args:string[]) } }) } else { - tdevGet(pref + s.id + "/text?original=true", text => { + tdevGet(pref + s.id + "/text?original=true" + suff, text => { if (!text) { console.log("ERROR", s.id) oneup() - } else if (/^meta hasIds "yes"/m.test(text)) { + } else if (liteMode || /^meta hasIds "yes"/m.test(text)) { fs.writeFileSync("dls/" + s.id, text) oneup() } else {