parse.y: escaped newline in dedenting heredoc

* parse.y (parser_here_document): an escaped newline is not an
  actual newline, and the rest part should not be dedented.
  [ruby-core:72855] [Bug #11989]

git-svn-id: svn+ssh://ci.ruby-lang.org/ruby/trunk@53573 b2dd03c8-39d4-4d8f-98ff-823fe69b080e

ChangeLog

@@ -1,3 +1,9 @@
+Mon Jan 18 14:37:07 2016  Nobuyoshi Nakada  <nobu@ruby-lang.org>
+
+	* parse.y (parser_here_document): an escaped newline is not an
+	  actual newline, and the rest part should not be dedented.
+	  [ruby-core:72855] [Bug #11989]
+
 Mon Jan 18 12:04:34 2016  SHIBATA Hiroshi  <hsbt@ruby-lang.org>
 
 	* test/ruby/test_string.rb: Added extra testcase for test_rstrip_bang

parse.y

@@ -6611,10 +6611,13 @@ parser_heredoc_restore(struct parser_params *parser, NODE *here)
 }
 
 static int
-dedent_pos(const char *str, long len, int width)
+dedent_string(VALUE string, int width)
 {
+    char *str;
+    long len;
     int i, col = 0;
 
+    RSTRING_GETMEM(string, str, len);
     for (i = 0; i < len && col < width; i++) {
 	if (str[i] == ' ') {
 	    col++;
@@ -6628,65 +6631,38 @@ dedent_pos(const char *str, long len, int width)
 	    break;
 	}
     }
+    MEMMOVE(str, str + i, char, len - i);
+    rb_str_set_len(string, len - i);
     return i;
 }
 
 #ifndef RIPPER
-static VALUE
-parser_heredoc_dedent_string(VALUE input, int width, int first)
-{
-    long len;
-    int col;
-    char *str, *p, *out_p, *end, *t;
-
-    RSTRING_GETMEM(input, str, len);
-    end = &str[len];
-
-    p = str;
-    if (!first) {
-	p = memchr(p, '\n', end - p);
-	if (!p) return input;
-	p++;
-    }
-    out_p = p;
-    while (p < end) {
-	col = dedent_pos(p, end - p, width);
-	p += col;
-	if (!(t = memchr(p, '\n', end - p)))
-	    t = end;
-	else
-	    ++t;
-	if (p > out_p) memmove(out_p, p, t - p);
-	out_p += t - p;
-	p = t;
-    }
-    rb_str_set_len(input, out_p - str);
-
-    return input;
-}
-
 static void
 parser_heredoc_dedent(struct parser_params *parser, NODE *root)
 {
     NODE *node, *str_node;
-    int first = TRUE;
+    int bol = TRUE;
     int indent = heredoc_indent;
 
     if (indent <= 0) return;
 
     node = str_node = root;
 
+    if (!root) return;
+    if (nd_type(root) == NODE_ARRAY) str_node = root->nd_head;
+
     while (str_node) {
 	VALUE lit = str_node->nd_lit;
-	if (NIL_P(parser_heredoc_dedent_string(lit, indent, first)))
+	if (bol) dedent_string(lit, indent);
-	    compile_error(PARSER_ARG "dedent failure: %d: %"PRIsVALUE, indent, lit);
+	bol = TRUE;
-	first = FALSE;
 
 	str_node = 0;
 	while ((node = node->nd_next) != 0 && nd_type(node) == NODE_ARRAY) {
 	    if ((str_node = node->nd_head) != 0) {
 		enum node_type type = nd_type(str_node);
 		if (type == NODE_STR || type == NODE_DSTR) break;
+		bol = FALSE;
+		str_node = 0;
 	    }
 	}
     }
@@ -6703,17 +6679,12 @@ parser_heredoc_dedent(struct parser_params *parser, VALUE array)
 static VALUE
 parser_dedent_string(VALUE self, VALUE input, VALUE width)
 {
-    char *str;
-    long len;
     int wid, col;
 
     StringValue(input);
     wid = NUM2UINT(width);
     rb_str_modify(input);
-    RSTRING_GETMEM(input, str, len);
+    col = dedent_string(input, wid);
-    col = dedent_pos(str, len, wid);
-    MEMMOVE(str, str + col, char, len - col);
-    rb_str_set_len(input, len - col);
     return INT2NUM(col);
 }
 #endif
@@ -6869,15 +6840,7 @@ parser_here_document(struct parser_params *parser, NODE *here)
     }
 
     if (!(func & STR_FUNC_EXPAND)) {
-	int end = 0;
 	do {
-#ifdef RIPPER
-	    if (end && heredoc_indent > 0) {
-		set_yylval_str(str);
-		flush_string_content(enc);
-		return tSTRING_CONTENT;
-	    }
-#endif
 	    p = RSTRING_PTR(lex_lastline);
 	    pend = lex_pend;
 	    if (pend > p) {
@@ -6905,6 +6868,11 @@ parser_here_document(struct parser_params *parser, NODE *here)
 		str = STR_NEW(p, pend - p);
 	    if (pend < lex_pend) rb_str_cat(str, "\n", 1);
 	    lex_goto_eol(parser);
+	    if (heredoc_indent > 0) {
+		set_yylval_str(str);
+		flush_string_content(enc);
+		return tSTRING_CONTENT;
+	    }
 	    if (nextc() == -1) {
 		if (str) {
 		    dispose_string(str);
@@ -6912,7 +6880,7 @@ parser_here_document(struct parser_params *parser, NODE *here)
 		}
 		goto error;
 	    }
-	} while (!(end = whole_match_p(eos, len, indent)));
+	} while (!whole_match_p(eos, len, indent));
     }
     else {
 	/*	int mb = ENC_CODERANGE_7BIT, *mbp = &mb;*/
@@ -6930,20 +6898,16 @@ parser_here_document(struct parser_params *parser, NODE *here)
 		goto restore;
 	    }
 	    if (c != '\n') {
-#ifdef RIPPER
 	      flush:
-#endif
 		set_yylval_str(STR_NEW3(tok(), toklen(), enc, func));
 		flush_string_content(enc);
 		return tSTRING_CONTENT;
 	    }
 	    tokadd(nextc());
-#ifdef RIPPER
+	    if (heredoc_indent > 0) {
-	    if (c == '\n' && heredoc_indent > 0) {
 		lex_goto_eol(parser);
 		goto flush;
 	    }
-#endif
 	    /*	    if (mbp && mb == ENC_CODERANGE_UNKNOWN) mbp = 0;*/
 	    if ((c = nextc()) == -1) goto error;
 	} while (!whole_match_p(eos, len, indent));
@@ -8975,6 +8939,16 @@ literal_concat_gen(struct parser_params *parser, NODE *head, NODE *tail)
 	head = list_append(node, head);
 	htype = NODE_DSTR;
     }
+    if (heredoc_indent > 0) {
+	switch (htype) {
+	  case NODE_STR:
+	    nd_set_type(head, NODE_DSTR);
+	  case NODE_DSTR:
+	    return list_append(head, tail);
+	  default:
+	    break;
+	}
+    }
     switch (nd_type(tail)) {
       case NODE_STR:
 	if (htype == NODE_DSTR && (headlast = head->nd_next->nd_end->nd_head) &&

test/ruby/test_syntax.rb

@@ -591,6 +591,15 @@ e"
     assert_dedented_heredoc(expect, result)
   end
 
+  def test_dedented_heredoc_with_newline
+    bug11989 = '[ruby-core:72855] [Bug #11989] after escaped newline should not be dedented'
+    result = '  x\n'"  y\n" \
+             "  z\n"
+    expect = 'x\n'"  y\n" \
+             "z\n"
+    assert_dedented_heredoc(expect, result, bug11989)
+  end
+
   def test_dedented_heredoc_with_concatenation
     bug11990 = '[ruby-core:72857] [Bug #11990] concatenated string should not be dedented'
     %w[eos "eos" 'eos'].each do |eos|