Bug 904144 - Parse "//# sourceURL=..." directives and expose them on Debugger.Source; r=jimb

2013-09-20 14:57:09 -07:00 · 2013-09-20 14:57:09 -07:00 · cded1e9685
--- a/js/src/frontend/BytecodeCompiler.cpp
+++ b/js/src/frontend/BytecodeCompiler.cpp
@ -39,6 +39,16 @@ CheckLength(ExclusiveContext *cx, size_t length)
    return true;
 }

+static bool
+SetSourceURL(ExclusiveContext *cx, TokenStream &tokenStream, ScriptSource *ss)
+{
+    if (tokenStream.hasSourceURL()) {
+        if (!ss->setSourceURL(cx, tokenStream.sourceURL()))
+            return false;
+    }
+    return true;
+}
+
 static bool
 SetSourceMap(ExclusiveContext *cx, TokenStream &tokenStream, ScriptSource *ss)
 {
@ -357,6 +367,9 @@ frontend::CompileScript(ExclusiveContext *cx, LifoAlloc *alloc, HandleObject sco
    if (!MaybeCheckEvalFreeVariables(cx, evalCaller, scopeChain, parser, pc.ref()))
        return NULL;

+    if (!SetSourceURL(cx, parser.tokenStream, ss))
+        return NULL;
+
    if (!SetSourceMap(cx, parser.tokenStream, ss))
        return NULL;

@ -578,6 +591,9 @@ CompileFunctionBody(JSContext *cx, MutableHandleFunction fun, CompileOptions opt
        JS_ASSERT(IsAsmJSModuleNative(fun->native()));
    }

+    if (!SetSourceURL(cx, parser.tokenStream, ss))
+        return false;
+
    if (!SetSourceMap(cx, parser.tokenStream, ss))
        return false;

--- a/js/src/frontend/TokenStream.cpp
+++ b/js/src/frontend/TokenStream.cpp
@ -274,6 +274,7 @@ TokenStream::TokenStream(ExclusiveContext *cx, const CompileOptions &options,
    prevLinebase(NULL),
    userbuf(cx, base - options.column, length + options.column), // See comment below
    filename(options.filename),
+    sourceURL_(NULL),
    sourceMapURL_(NULL),
    tokenbuf(cx),
    cx(cx),
@ -333,6 +334,7 @@ TokenStream::TokenStream(ExclusiveContext *cx, const CompileOptions &options,

 TokenStream::~TokenStream()
 {
+    js_free(sourceURL_);
    js_free(sourceMapURL_);

    JS_ASSERT_IF(originPrincipals, originPrincipals->refcount);
@ -807,31 +809,45 @@ CharsMatch(const jschar *p, const char *q) {
 }

 bool
-TokenStream::getSourceMappingURL(bool isMultiline, bool shouldWarnDeprecated)
+TokenStream::getDirectives(bool isMultiline, bool shouldWarnDeprecated)
 {
-    // Match comments of the form "//# sourceMappingURL=<url>" or
-    // "/\* //# sourceMappingURL=<url> *\/"
+    // Match directive comments used in debugging, such as "//# sourceURL" and
+    // "//# sourceMappingURL". Use of "//@" instead of "//#" is deprecated.
    //
    // To avoid a crashing bug in IE, several JavaScript transpilers wrap single
    // line comments containing a source mapping URL inside a multiline
    // comment. To avoid potentially expensive lookahead and backtracking, we
    // only check for this case if we encounter a '#' character.
+
+    if (!getSourceURL(isMultiline, shouldWarnDeprecated))
+        return false;
+    if (!getSourceMappingURL(isMultiline, shouldWarnDeprecated))
+        return false;
+
+    return true;
+}
+
+bool
+TokenStream::getDirective(bool isMultiline, bool shouldWarnDeprecated,
+                          const char *directive, int directiveLength,
+                          const char *errorMsgPragma, jschar **destination) {
+    JS_ASSERT(directiveLength <= 18);
    jschar peeked[18];
    int32_t c;

-    if (peekChars(18, peeked) && CharsMatch(peeked, " sourceMappingURL=")) {
-        if (shouldWarnDeprecated && !reportWarning(JSMSG_DEPRECATED_SOURCE_MAP)) {
+    if (peekChars(directiveLength, peeked) && CharsMatch(peeked, directive)) {
+        if (shouldWarnDeprecated &&
+            !reportWarning(JSMSG_DEPRECATED_PRAGMA, errorMsgPragma))
            return false;
-        }

-        skipChars(18);
+        skipChars(directiveLength);
        tokenbuf.clear();

        while ((c = peekChar()) && c != EOF && !IsSpaceOrBOM2(c)) {
            getChar();
-            // Source mapping URLs can occur in both single- and multiline
-            // comments. If we're currently inside a multiline comment, we also
-            // need to recognize multiline comment terminators.
+            // Debugging directives can occur in both single- and multi-line
+            // comments. If we're currently inside a multi-line comment, we also
+            // need to recognize multi-line comment terminators.
            if (isMultiline && c == '*' && peekChar() == '/') {
                ungetChar('*');
                break;
@ -840,23 +856,44 @@ TokenStream::getSourceMappingURL(bool isMultiline, bool shouldWarnDeprecated)
        }

        if (tokenbuf.empty())
-            // The source map's URL was missing, but not quite an exception that
-            // we should stop and drop everything for, though.
+            // The directive's URL was missing, but this is not quite an
+            // exception that we should stop and drop everything for.
            return true;

-        size_t sourceMapURLLength = tokenbuf.length();
+        size_t length = tokenbuf.length();

-        js_free(sourceMapURL_);
-        sourceMapURL_ = cx->pod_malloc<jschar>(sourceMapURLLength + 1);
-        if (!sourceMapURL_)
+        js_free(*destination);
+        *destination = cx->pod_malloc<jschar>(length + 1);
+        if (!*destination)
            return false;

-        PodCopy(sourceMapURL_, tokenbuf.begin(), sourceMapURLLength);
-        sourceMapURL_[sourceMapURLLength] = '\0';
+        PodCopy(*destination, tokenbuf.begin(), length);
+        (*destination)[length] = '\0';
    }
+
    return true;
 }

+bool
+TokenStream::getSourceURL(bool isMultiline, bool shouldWarnDeprecated)
+{
+    // Match comments of the form "//# sourceURL=<url>" or
+    // "/\* //# sourceURL=<url> *\/"
+
+    return getDirective(isMultiline, shouldWarnDeprecated, " sourceURL=", 11,
+                        "sourceURL", &sourceURL_);
+}
+
+bool
+TokenStream::getSourceMappingURL(bool isMultiline, bool shouldWarnDeprecated)
+{
+    // Match comments of the form "//# sourceMappingURL=<url>" or
+    // "/\* //# sourceMappingURL=<url> *\/"
+
+    return getDirective(isMultiline, shouldWarnDeprecated, " sourceMappingURL=", 18,
+                        "sourceMappingURL", &sourceMapURL_);
+}
+
 JS_ALWAYS_INLINE Token *
 TokenStream::newToken(ptrdiff_t adjust)
 {
@ -1506,7 +1543,8 @@ TokenStream::getTokenInternal(Modifier modifier)
        if (matchChar('/')) {
            c = peekChar();
            if (c == '@' || c == '#') {
-                if (!getSourceMappingURL(false, getChar() == '@'))
+                bool shouldWarn = getChar() == '@';
+                if (!getDirectives(false, shouldWarn))
                    goto error;
            }

@ -1524,7 +1562,8 @@ TokenStream::getTokenInternal(Modifier modifier)
            while ((c = getChar()) != EOF &&
                   !(c == '*' && matchChar('/'))) {
                if (c == '@' || c == '#') {
-                    if (!getSourceMappingURL(true, c == '@'))
+                    bool shouldWarn = c == '@';
+                    if (!getDirectives(true, shouldWarn))
                        goto error;
                }
            }
--- a/js/src/frontend/TokenStream.h
+++ b/js/src/frontend/TokenStream.h
@ -596,6 +596,14 @@ class MOZ_STACK_CLASS TokenStream
        return pos.buf - userbuf.base();
    }

+    bool hasSourceURL() const {
+        return sourceURL_ != NULL;
+    }
+
+    jschar *sourceURL() {
+        return sourceURL_;
+    }
+
    bool hasSourceMapURL() const {
        return sourceMapURL_ != NULL;
    }
@ -803,6 +811,12 @@ class MOZ_STACK_CLASS TokenStream
    bool matchUnicodeEscapeIdStart(int32_t *c);
    bool matchUnicodeEscapeIdent(int32_t *c);
    bool peekChars(int n, jschar *cp);
+
+    bool getDirectives(bool isMultiline, bool shouldWarnDeprecated);
+    bool getDirective(bool isMultiline, bool shouldWarnDeprecated,
+                      const char *directive, int directiveLength,
+                      const char *errorMsgPragma, jschar **destination);
+    bool getSourceURL(bool isMultiline, bool shouldWarnDeprecated);
    bool getSourceMappingURL(bool isMultiline, bool shouldWarnDeprecated);

    // |expect| cannot be an EOL char.
@ -843,6 +857,7 @@ class MOZ_STACK_CLASS TokenStream
    const jschar        *prevLinebase;      // start of previous line;  NULL if on the first line
    TokenBuf            userbuf;            // user input buffer
    const char          *filename;          // input filename or null
+    jschar              *sourceURL_;        // the user's requested source URL or null
    jschar              *sourceMapURL_;     // source map's filename or null
    CharBuffer          tokenbuf;           // current token string buffer
    bool                maybeEOL[256];      // probabilistic EOL lookup table
--- a/js/src/jit-test/tests/debug/Source-displayURL-deprecated.js
+++ b/js/src/jit-test/tests/debug/Source-displayURL-deprecated.js
@ -0,0 +1,26 @@
+/* -*- Mode: javascript; js-indent-level: 4; -*- */
+// Source.prototype.sourceURL can be a string or null.
+
+let g = newGlobal('new-compartment');
+let dbg = new Debugger;
+let gw = dbg.addDebuggee(g);
+
+function getDisplayURL() {
+    let fw = gw.makeDebuggeeValue(g.f);
+    return fw.script.source.displayURL;
+}
+
+// Comment pragmas
+g.evaluate('function f() {}\n' +
+           '//@ sourceURL=file:///var/quux.js');
+assertEq(getDisplayURL(), 'file:///var/quux.js');
+
+g.evaluate('function f() {}\n' +
+           '/*//@ sourceURL=file:///var/quux.js*/');
+assertEq(getDisplayURL(), 'file:///var/quux.js');
+
+g.evaluate('function f() {}\n' +
+           '/*\n' +
+           '//@ sourceURL=file:///var/quux.js\n' +
+           '*/');
+assertEq(getDisplayURL(), 'file:///var/quux.js');
--- a/js/src/jit-test/tests/debug/Source-displayURL.js
+++ b/js/src/jit-test/tests/debug/Source-displayURL.js
@ -0,0 +1,71 @@
+/* -*- Mode: javascript; js-indent-level: 4; -*- */
+// Source.prototype.sourceURL can be a string or null.
+
+let g = newGlobal('new-compartment');
+let dbg = new Debugger;
+let gw = dbg.addDebuggee(g);
+
+function getDisplayURL() {
+    let fw = gw.makeDebuggeeValue(g.f);
+    return fw.script.source.displayURL;
+}
+
+// Without a source url
+g.evaluate("function f(x) { return 2*x; }");
+assertEq(getDisplayURL(), null);
+
+// With a source url
+g.evaluate("function f(x) { return 2*x; }", {sourceURL: 'file:///var/foo.js'});
+assertEq(getDisplayURL(), 'file:///var/foo.js');
+
+// Nested functions
+let fired = false;
+dbg.onDebuggerStatement = function (frame) {
+    fired = true;
+    assertEq(frame.script.source.displayURL, 'file:///var/bar.js');
+};
+g.evaluate('(function () { (function () { debugger; })(); })();',
+           {sourceURL: 'file:///var/bar.js'});
+assertEq(fired, true);
+
+// Comment pragmas
+g.evaluate('function f() {}\n' +
+           '//# sourceURL=file:///var/quux.js');
+assertEq(getDisplayURL(), 'file:///var/quux.js');
+
+g.evaluate('function f() {}\n' +
+           '/*//# sourceURL=file:///var/quux.js*/');
+assertEq(getDisplayURL(), 'file:///var/quux.js');
+
+g.evaluate('function f() {}\n' +
+           '/*\n' +
+           '//# sourceURL=file:///var/quux.js\n' +
+           '*/');
+assertEq(getDisplayURL(), 'file:///var/quux.js');
+
+// Spaces are disallowed by the URL spec (they should have been
+// percent-encoded).
+g.evaluate('function f() {}\n' +
+           '//# sourceURL=http://example.com/has illegal spaces');
+assertEq(getDisplayURL(), 'http://example.com/has');
+
+// When the URL is missing, we don't set the sourceMapURL and we don't skip the
+// next line of input.
+g.evaluate('function f() {}\n' +
+           '//# sourceURL=\n' +
+           'function z() {}');
+assertEq(getDisplayURL(), null);
+assertEq('z' in g, true);
+
+// The last comment pragma we see should be the one which sets the displayURL.
+g.evaluate('function f() {}\n' +
+           '//# sourceURL=http://example.com/foo.js\n' +
+           '//# sourceURL=http://example.com/bar.js');
+assertEq(getDisplayURL(), 'http://example.com/bar.js');
+
+// With both a comment and the evaluate option.
+g.evaluate('function f() {}\n' +
+           '//# sourceURL=http://example.com/foo.js',
+           {sourceMapURL: 'http://example.com/bar.js'});
+assertEq(getDisplayURL(), 'http://example.com/foo.js');
+
--- a/js/src/js.msg
+++ b/js/src/js.msg
@ -353,7 +353,7 @@ MSG_DEF(JSMSG_REST_WITH_DEFAULT,      299, 0, JSEXN_SYNTAXERR, "rest parameter m
 MSG_DEF(JSMSG_NONDEFAULT_FORMAL_AFTER_DEFAULT, 300, 0, JSEXN_SYNTAXERR, "parameter(s) with default followed by parameter without default")
 MSG_DEF(JSMSG_YIELD_IN_DEFAULT,       301, 0, JSEXN_SYNTAXERR, "yield in default expression")
 MSG_DEF(JSMSG_INTRINSIC_NOT_DEFINED,  302, 1, JSEXN_REFERENCEERR, "no intrinsic function {0}")
-MSG_DEF(JSMSG_ALREADY_HAS_SOURCE_MAP_URL, 303, 1, JSEXN_ERR,      "{0} is being assigned a source map URL, but already has one")
+MSG_DEF(JSMSG_ALREADY_HAS_PRAGMA, 303, 2, JSEXN_ERR,      "{0} is being assigned a {1}, but already has one")
 MSG_DEF(JSMSG_PAR_ARRAY_BAD_ARG,      304, 1, JSEXN_RANGEERR, "invalid ParallelArray{0} argument")
 MSG_DEF(JSMSG_PAR_ARRAY_BAD_PARTITION, 305, 0, JSEXN_ERR, "argument must be divisible by outermost dimension")
 MSG_DEF(JSMSG_PAR_ARRAY_REDUCE_EMPTY, 306, 0, JSEXN_ERR, "cannot reduce ParallelArray object whose outermost dimension is empty")
@ -400,7 +400,7 @@ MSG_DEF(JSMSG_YIELD_IN_ARROW,         346, 0, JSEXN_SYNTAXERR, "arrow function m
 MSG_DEF(JSMSG_WRONG_VALUE,            347, 2, JSEXN_ERR, "expected {0} but found {1}")
 MSG_DEF(JSMSG_PAR_ARRAY_SCATTER_BAD_TARGET, 348, 1, JSEXN_ERR, "target for index {0} is not an integer")
 MSG_DEF(JSMSG_SELFHOSTED_UNBOUND_NAME,349, 0, JSEXN_TYPEERR, "self-hosted code may not contain unbound name lookups")
-MSG_DEF(JSMSG_DEPRECATED_SOURCE_MAP,  350, 0, JSEXN_SYNTAXERR, "Using //@ to indicate source map URL pragmas is deprecated. Use //# instead")
+MSG_DEF(JSMSG_DEPRECATED_PRAGMA,  350, 1, JSEXN_SYNTAXERR, "Using //@ to indicate {0} pragmas is deprecated. Use //# instead")
 MSG_DEF(JSMSG_BAD_DESTRUCT_ASSIGN,    351, 1, JSEXN_SYNTAXERR, "can't assign to {0} using destructuring assignment")
 MSG_DEF(JSMSG_TYPEDOBJECT_ARRAYTYPE_BAD_ARGS, 352, 0, JSEXN_ERR, "Invalid arguments")
 MSG_DEF(JSMSG_TYPEDOBJECT_BINARYARRAY_BAD_INDEX, 353, 0, JSEXN_RANGEERR, "invalid or out-of-range index")
--- a/js/src/jsscript.cpp
+++ b/js/src/jsscript.cpp
@ -1208,6 +1208,7 @@ ScriptSource::destroy()
    JS_ASSERT(ready());
    adjustDataSize(0);
    js_free(filename_);
+    js_free(sourceURL_);
    js_free(sourceMapURL_);
    if (originPrincipals_)
        JS_DropPrincipals(TlsPerThreadData.get()->runtimeFromMainThread(), originPrincipals_);
@ -1298,6 +1299,31 @@ ScriptSource::performXDR(XDRState<mode> *xdr)
        sourceMapURL_[sourceMapURLLen] = '\0';
    }

+    uint8_t haveSourceURL = hasSourceURL();
+    if (!xdr->codeUint8(&haveSourceURL))
+        return false;
+
+    if (haveSourceURL) {
+        uint32_t sourceURLLen = (mode == XDR_DECODE) ? 0 : js_strlen(sourceURL_);
+        if (!xdr->codeUint32(&sourceURLLen))
+            return false;
+
+        if (mode == XDR_DECODE) {
+            size_t byteLen = (sourceURLLen + 1) * sizeof(jschar);
+            sourceURL_ = static_cast<jschar *>(xdr->cx()->malloc_(byteLen));
+            if (!sourceURL_)
+                return false;
+        }
+        if (!xdr->codeChars(sourceURL_, sourceURLLen)) {
+            if (mode == XDR_DECODE) {
+                js_free(sourceURL_);
+                sourceURL_ = NULL;
+            }
+            return false;
+        }
+        sourceURL_[sourceURLLen] = '\0';
+    }
+
    uint8_t haveFilename = !!filename_;
    if (!xdr->codeUint8(&haveFilename))
        return false;
@ -1330,21 +1356,53 @@ ScriptSource::setFilename(ExclusiveContext *cx, const char *filename)
    return true;
 }

+bool
+ScriptSource::setSourceURL(ExclusiveContext *cx, const jschar *sourceURL)
+{
+    JS_ASSERT(sourceURL);
+    if (hasSourceURL()) {
+        if (cx->isJSContext() &&
+            !JS_ReportErrorFlagsAndNumber(cx->asJSContext(), JSREPORT_WARNING,
+                                          js_GetErrorMessage, NULL,
+                                          JSMSG_ALREADY_HAS_PRAGMA, filename_,
+                                          "//# sourceURL"))
+        {
+            return false;
+        }
+    }
+    size_t len = js_strlen(sourceURL) + 1;
+    if (len == 1)
+        return true;
+    sourceURL_ = js_strdup(cx, sourceURL);
+    if (!sourceURL_)
+        return false;
+    return true;
+}
+
+const jschar *
+ScriptSource::sourceURL()
+{
+    JS_ASSERT(hasSourceURL());
+    return sourceURL_;
+}
+
 bool
 ScriptSource::setSourceMapURL(ExclusiveContext *cx, const jschar *sourceMapURL)
 {
    JS_ASSERT(sourceMapURL);
    if (hasSourceMapURL()) {
        if (cx->isJSContext() &&
-            !JS_ReportErrorFlagsAndNumber(cx->asJSContext(), JSREPORT_WARNING, js_GetErrorMessage,
-                                          NULL, JSMSG_ALREADY_HAS_SOURCE_MAP_URL, filename_))
+            !JS_ReportErrorFlagsAndNumber(cx->asJSContext(), JSREPORT_WARNING,
+                                          js_GetErrorMessage, NULL,
+                                          JSMSG_ALREADY_HAS_PRAGMA, filename_,
+                                          "//# sourceMappingURL"))
        {
            return false;
        }
    }

    size_t len = js_strlen(sourceMapURL) + 1;
-    if (len == 1) 
+    if (len == 1)
        return true;
    sourceMapURL_ = js_strdup(cx, sourceMapURL);
    if (!sourceMapURL_)
--- a/js/src/jsscript.h
+++ b/js/src/jsscript.h
@ -304,6 +304,7 @@ class ScriptSource
    uint32_t length_;
    uint32_t compressedLength_;
    char *filename_;
+    jschar *sourceURL_;
    jschar *sourceMapURL_;
    JSPrincipals *originPrincipals_;

@ -320,6 +321,7 @@ class ScriptSource
        length_(0),
        compressedLength_(0),
        filename_(NULL),
+        sourceURL_(NULL),
        sourceMapURL_(NULL),
        originPrincipals_(originPrincipals),
        sourceRetrievable_(false),
@ -367,6 +369,11 @@ class ScriptSource
        return filename_;
    }

+    // Source URLs
+    bool setSourceURL(ExclusiveContext *cx, const jschar *sourceURL);
+    const jschar *sourceURL();
+    bool hasSourceURL() const { return sourceURL_ != NULL; }
+
    // Source maps
    bool setSourceMapURL(ExclusiveContext *cx, const jschar *sourceMapURL);
    const jschar *sourceMapURL();
--- a/js/src/shell/js.cpp
+++ b/js/src/shell/js.cpp
@ -906,6 +906,7 @@ Evaluate(JSContext *cx, unsigned argc, jsval *vp)
    const char *fileName = "@evaluate";
    RootedObject element(cx);
    JSAutoByteString fileNameBytes;
+    RootedString sourceURL(cx);
    RootedString sourceMapURL(cx);
    unsigned lineNumber = 1;
    RootedObject global(cx, NULL);
@ -966,6 +967,14 @@ Evaluate(JSContext *cx, unsigned argc, jsval *vp)
        if (!JSVAL_IS_PRIMITIVE(v))
            element = JSVAL_TO_OBJECT(v);

+        if (!JS_GetProperty(cx, opts, "sourceURL", &v))
+            return false;
+        if (!JSVAL_IS_VOID(v)) {
+            sourceURL = JS_ValueToString(cx, v);
+            if (!sourceURL)
+                return false;
+        }
+
        if (!JS_GetProperty(cx, opts, "sourceMapURL", &v))
            return false;
        if (!JSVAL_IS_VOID(v)) {
@ -1054,6 +1063,13 @@ Evaluate(JSContext *cx, unsigned argc, jsval *vp)
        if (!script)
            return false;

+        if (sourceURL && !script->scriptSource()->hasSourceURL()) {
+            const jschar *surl = JS_GetStringCharsZ(cx, sourceURL);
+            if (!surl)
+                return false;
+            if (!script->scriptSource()->setSourceURL(cx, surl))
+                return false;
+        }
        if (sourceMapURL && !script->scriptSource()->hasSourceMapURL()) {
            const jschar *smurl = JS_GetStringCharsZ(cx, sourceMapURL);
            if (!smurl)
--- a/js/src/vm/Debugger.cpp
+++ b/js/src/vm/Debugger.cpp
@ -3753,9 +3753,30 @@ DebuggerSource_getUrl(JSContext *cx, unsigned argc, Value *vp)
    return true;
 }

+static bool
+DebuggerSource_getDisplayURL(JSContext *cx, unsigned argc, Value *vp)
+{
+    THIS_DEBUGSOURCE_REFERENT(cx, argc, vp, "(get url)", args, obj, sourceObject);
+
+    ScriptSource *ss = sourceObject->source();
+    JS_ASSERT(ss);
+
+    if (ss->hasSourceURL()) {
+        JSString *str = JS_NewUCStringCopyZ(cx, ss->sourceURL());
+        if (!str)
+            return false;
+        args.rval().setString(str);
+    } else {
+        args.rval().setNull();
+    }
+
+    return true;
+}
+
 static const JSPropertySpec DebuggerSource_properties[] = {
    JS_PSG("text", DebuggerSource_getText, 0),
    JS_PSG("url", DebuggerSource_getUrl, 0),
+    JS_PSG("displayURL", DebuggerSource_getDisplayURL, 0),
    JS_PS_END
 };