* parse.y (yylex): should dispatch scan-event even when follows

just after delayed-token.  [ruby-dev:37855] [Bug #1071]


git-svn-id: svn+ssh://ci.ruby-lang.org/ruby/trunk@24557 b2dd03c8-39d4-4d8f-98ff-823fe69b080e

ChangeLog

@@ -1,3 +1,8 @@
+Sun Aug 16 23:58:22 2009  Nobuyoshi Nakada  <nobu@ruby-lang.org>
+
+	* parse.y (yylex): should dispatch scan-event even when follows
+	  just after delayed-token.  [ruby-dev:37855] [Bug #1071]
+
 Sun Aug 16 22:20:16 2009  Tadayoshi Funaba  <tadf@dotrb.org>
 
 	* lib/date/format.rb: reverted.

parse.y

@@ -7612,7 +7612,6 @@ yylex(void *p)
 #ifdef RIPPER
     if (!NIL_P(parser->delayed)) {
 	ripper_dispatch_delayed_token(parser, t);
-	return t;
     }
     if (t != 0)
 	ripper_dispatch_scan_event(parser, t);

test/ripper/test_scanner_events.rb

@@ -63,6 +63,11 @@ class TestRipper_ScannerEvents < Test::Unit::TestCase
                   [[2, 0], :on_tstring_content, "heredoc\n"],
                   [[3, 0], :on_heredoc_end, "EOS"]],
                  Ripper.lex("<<EOS\nheredoc\nEOS")
+    assert_equal [[[1, 0], :on_regexp_beg, "/"],
+                  [[1, 1], :on_tstring_content, "foo\n"],
+                  [[2, 0], :on_tstring_content, "bar"],
+                  [[2, 3], :on_regexp_end, "/"]],
+                 Ripper.lex("/foo\nbar/")
   end
 
   def test_location