[Bug #19312] Return end-of-input at `__END__`

parse.y

@@ -9813,7 +9813,7 @@ parser_yylex(struct parser_params *p)
 #endif
 	/* Set location for end-of-input because dispatch_scan_event is not called. */
 	RUBY_SET_YYLLOC(*p->yylloc);
-	return 0;
+	return END_OF_INPUT;
 
 	/* white spaces */
       case '\r':
@@ -9986,7 +9986,7 @@ parser_yylex(struct parser_params *p)
 		    c = nextc(p);
 		    if (c == -1) {
 			compile_error(p, "embedded document meets end of file");
-			return 0;
+			return END_OF_INPUT;
 		    }
 		    if (c == '=' && word_match_p(p, "end", 3)) {
 			break;
@@ -10466,13 +10466,11 @@ parser_yylex(struct parser_params *p)
 	if (was_bol(p) && whole_match_p(p, "__END__", 7, 0)) {
 	    p->ruby__end__seen = 1;
 	    p->eofp = 1;
-#ifndef RIPPER
-	    return -1;
-#else
+#ifdef RIPPER
             lex_goto_eol(p);
             dispatch_scan_event(p, k__END__);
-            return 0;
 #endif
+            return END_OF_INPUT;
 	}
 	newtok(p);
 	break;
@@ -10504,7 +10502,7 @@ yylex(YYSTYPE *lval, YYLTYPE *yylloc, struct parser_params *p)
 
     if (has_delayed_token(p))
 	dispatch_delayed_token(p, t);
-    else if (t != 0)
+    else if (t != END_OF_INPUT)
 	dispatch_scan_event(p, t);
 
     return t;

test/ruby/test_ast.rb

@@ -624,6 +624,27 @@ dummy
     assert_equal("def test_keep_script_lines_for_of\n", node_method.source.lines.first)
   end
 
+  def test_keep_tokens_for_parse
+    node = RubyVM::AbstractSyntaxTree.parse(<<~END, keep_tokens: true)
+    1.times do
+    end
+    __END__
+    dummy
+    END
+
+    expected = [
+      [:tINTEGER, "1"],
+      [:".", "."],
+      [:tIDENTIFIER, "times"],
+      [:tSP, " "],
+      [:keyword_do, "do"],
+      [:tIGNORED_NL, "\n"],
+      [:keyword_end, "end"],
+      [:nl, "\n"],
+    ]
+    assert_equal(expected, node.all_tokens.map { [_2, _3]})
+  end
+
   def test_encoding_with_keep_script_lines
     # Stop a warning "possibly useless use of a literal in void context"
     verbose_bak, $VERBOSE = $VERBOSE, nil