* test/ripper/test_scanner_events.rb: tokens must be reordered.

* ext/ripper/lib/ripper/tokenizer.rb: ditto.


git-svn-id: svn+ssh://ci.ruby-lang.org/ruby/trunk@6937 b2dd03c8-39d4-4d8f-98ff-823fe69b080e

ChangeLog

@@ -1,3 +1,9 @@
+Mon Sep 20 17:38:43 2004  Minero Aoki  <aamine@loveruby.net>
+
+	* test/ripper/test_scanner_events.rb: tokens must be reordered.
+
+	* ext/ripper/lib/ripper/tokenizer.rb: ditto.
+
 Mon Sep 20 16:58:16 2004  Minero Aoki  <aamine@loveruby.net>
 
 	* parse.y [ripper]: do not delay dispatching.

ext/ripper/lib/ripper/tokenizer.rb

@@ -16,22 +16,69 @@ class Ripper
     Tokenizer.tokenize(str)
   end
 
+
   class Tokenizer < ::Ripper
-    def Tokenizer.tokenize(str)
-      new(str).tokenize
+
+    def Tokenizer.tokenize(str, filename = '-', lineno = 1)
+      new(str, filename, lineno).tokenize
+    end
+
+    def initialize(src, filename = '-', lineno = 1)
+      @src = src
+      @__filename = filename
+      @__linestart = lineno
+      @__line = nil
+      @__col = nil
+    end
+
+    def filename
+      @__filename
+    end
+
+    def lineno
+      @__line
+    end
+
+    def column
+      @__col
     end
 
     def tokenize
-      @tokens = []
-      parse
-      @tokens.sort_by {|tok, pos| pos }.map {|tok,| tok }
+      _exec_tokenizer().map {|pos, event, tok| tok }
+    end
+
+    def parse
+      _exec_tokenizer().each do |pos, event, tok|
+        @__line, @__col = *pos
+        on__scan(event, tok)
+        __send__(event, tok)
+      end
+      data
     end
 
     private
 
-    def on__scan(type, tok)
-      @tokens.push [tok, [lineno(),column()]]
+    def _exec_tokenizer
+      TokenSorter.new(@src, @__filename, @__linestart).parse
     end
+
+  end
+
+
+  class TokenSorter < ::Ripper   #:nodoc: internal use only
+
+    def parse
+      @data = []
+      super
+      @data.sort_by {|pos, event, tok| pos }
+    end
+
+    private
+
+    def on__scan(event, tok)
+      @data.push [[lineno(),column()], event, tok]
+    end
+
   end
 
 end

test/ripper/test_scanner_events.rb

@@ -20,11 +20,11 @@ class TestRipper_ScannerEvents < Test::Unit::TestCase
     def parse
       @tokens = []
       super
-      @tokens
+      @tokens.sort_by {|tok,pos| pos }.map {|tok,| tok }
     end
 
     def on__scan(type, tok)
-      @tokens.push tok if !@target or type == @target
+      @tokens.push [tok,[lineno(),column()]] if !@target or type == @target
     end
   end
 
@@ -80,7 +80,8 @@ class TestRipper_ScannerEvents < Test::Unit::TestCase
   def validate_location(src)
     data = PosInfo.new(src).parse
     buf = ''
-    data.each do |tok, type, line, col|
+    data.sort_by {|tok, type, line, col| [line,col] }\
+        .each do |tok, type, line, col|
       assert_equal buf.count("\n") + 1, line,
           "wrong lineno: #{tok.inspect} (#{type}) [#{line}:#{col}]"
       assert_equal buf.sub(/\A.*\n/m, '').size, col,