Do not allow non-triple-quoted strings to be multi-line

packages/adl-language/src/spec.emu.html

@@ -111,15 +111,23 @@ BinaryDigits :
 BinaryDigit :
     one of `0` `1`
 
-// TODO: triple-quoted strings not specified yet, tricky to express.
-
+// NOTE: This does not specify the extra rules about '"""'s going
+//       on their own lines and having consistent indentation.
 StringLiteral :
     `"` StringCharacters? `"`
+    `"""` TripleQuotedStringCharacters? `"""`
 
 StringCharacters :
     StringCharacter StringCharacters?
 
 StringCharacter :
+    SourceCharacter but not one of `"` or `\` or LineTerminator
+    `\` EscapeCharacter
+
+TripleQuotedStringCharacters
+    TripleQuotedStringCharacter TripleQuotedStringCharacters?
+
+TripleQuotedStringCharacter :
     SourceCharacter but not one of `"` or `\`
     `\` EscapeCharacter
 

packages/adl-vscode/src/tmlanguage.ts

@@ -21,6 +21,7 @@ type ADLScope =
   | "entity.name.function.adl"
   | "keyword.other.adl"
   | "string.quoted.double.adl"
+  | "string.quoted.triple.adl"
   | "variable.name.adl";
 
 const meta: typeof tm.meta = tm.meta;
@@ -72,13 +73,19 @@ const escapeChar: MatchRule = {
   match: "\\\\.",
 };
 
-// TODO: Triple-quoted """X""" currently matches as three string literals
-// ("" "X" "") but should be its own thing.
 const stringLiteral: BeginEndRule = {
   key: "string-literal",
   scope: "string.quoted.double.adl",
   begin: '"',
-  end: '"',
+  end: '"|$',
+  patterns: [escapeChar],
+};
+
+const tripleQuotedStringLiteral: BeginEndRule = {
+  key: "triple-quoted-string-literal",
+  scope: "string.quoted.triple.adl",
+  begin: '"""',
+  end: '"""',
   patterns: [escapeChar],
 };
 
@@ -104,7 +111,16 @@ const blockComment: BeginEndRule = {
 // Tokens that match standing alone in any context: literals and comments
 const token: IncludeRule = {
   key: "token",
-  patterns: [lineComment, blockComment, stringLiteral, booleanLiteral, numericLiteral],
+  patterns: [
+    lineComment,
+    blockComment,
+    // `"""` must come before `"` or first two quotes of `"""` will match as
+    // empty string
+    tripleQuotedStringLiteral,
+    stringLiteral,
+    booleanLiteral,
+    numericLiteral,
+  ],
 };
 
 const parenthesizedExpression: BeginEndRule = {
@@ -181,7 +197,15 @@ const modelExpression: BeginEndRule = {
   scope: meta,
   begin: "\\{",
   end: "\\}",
-  patterns: [token, decorator, modelProperty, modelSpreadProperty],
+  patterns: [
+    // modelProperty must come before token or quoted property name will be
+    // considered an arbitrarily positioned string literal and not match as part
+    // of modelProperty begin.
+    modelProperty,
+    token,
+    decorator,
+    modelSpreadProperty,
+  ],
 };
 
 const modelHeritage: BeginEndRule = {

packages/adl/compiler/scanner.ts

@@ -205,10 +205,9 @@ export interface Scanner {
 
 const enum TokenFlags {
   None = 0,
-  HasCrlf = 1 << 0,
-  Escaped = 1 << 1,
-  TripleQuoted = 1 << 2,
-  Unterminated = 1 << 3,
+  Escaped = 1 << 0,
+  TripleQuoted = 1 << 1,
+  Unterminated = 1 << 2,
 }
 
 export function isLiteral(token: Token) {
@@ -582,12 +581,6 @@ export function createScanner(source: string | SourceFile, onError = throwOnErro
     loop: for (; !eof(); position++) {
       const ch = input.charCodeAt(position);
       switch (ch) {
-        case CharCode.CarriageReturn:
-          if (lookAhead(1) === CharCode.LineFeed) {
-            tokenFlags |= TokenFlags.HasCrlf;
-            position++;
-          }
-          break;
         case CharCode.Backslash:
           tokenFlags |= TokenFlags.Escaped;
           position++;
@@ -598,6 +591,14 @@ export function createScanner(source: string | SourceFile, onError = throwOnErro
         case CharCode.DoubleQuote:
           position++;
           return (token = Token.StringLiteral);
+        case CharCode.CarriageReturn:
+        case CharCode.LineFeed:
+          break loop;
+        default:
+          if (ch > CharCode.MaxAscii && isNonAsciiLineBreak(ch)) {
+            break loop;
+          }
+          continue;
       }
     }
 
@@ -635,11 +636,7 @@ export function createScanner(source: string | SourceFile, onError = throwOnErro
       return unescapeString(start, end);
     }
 
-    let value = input.substring(start, end);
-    if (tokenFlags & TokenFlags.HasCrlf) {
-      value = value.replace(/\r\n/g, "\n");
-    }
-    return value;
+    return input.substring(start, end);
   }
 
   function unindentAndUnescapeTripleQuotedString(start: number, end: number) {

packages/adl/test/test-parser.ts

@@ -226,7 +226,17 @@ describe("syntax", () => {
   describe("unterminated tokens", () => {
     parseErrorEach([["/* Yada yada yada", [/Unterminated multi-line comment/]]]);
 
-    const strings = ['"banana', '"banana\\', '"""\nbanana', '"""\nbanana\\'];
+    const strings = [
+      '"banana',
+      '"banana\\',
+      '"banana\r"',
+      '"banana\n"',
+      '"banana\r\n"',
+      '"banana\u{2028}"',
+      '"banana\u{2029}"',
+      '"""\nbanana',
+      '"""\nbanana\\',
+    ];
     parseErrorEach(
       Array.from(strings.entries()).map((e) => [
         `alias ${String.fromCharCode(CharCode.A + e[0])} = ${e[1]}`,

packages/adl/test/test-scanner.ts

@@ -206,8 +206,11 @@ describe("scanner", () => {
     scanString('"Hello world \\r\\n \\t \\" \\\\ !"', 'Hello world \r\n \t " \\ !');
   });
 
-  it("scans multi-line strings", () => {
-    scanString('"More\r\nthan\r\none\r\nline"', "More\nthan\none\nline");
+  it("does not allow multi-line, non-triple-quoted strings", () => {
+    scanString('"More\r\nthan\r\none\r\nline"', "More", /Unterminated string/);
+    scanString('"More\nthan\none\nline"', "More", /Unterminated string/);
+    scanString('"Fancy\u{2028}line separator"', "Fancy", /Unterminated string/);
+    scanString('"Fancy\u{2029}paragraph separator', "Fancy", /Unterminated string/);
   });
 
   it("scans triple-quoted strings", () => {