зеркало из https://github.com/github/ruby.git
* tool/transcode-tblgen.rb: distinguish UNDEF and INVALID.
[ruby-dev:35709] * transcode.c (transcode_loop): don't need rb_enc_mbclen now. git-svn-id: svn+ssh://ci.ruby-lang.org/ruby/trunk@18390 b2dd03c8-39d4-4d8f-98ff-823fe69b080e
This commit is contained in:
akr
Родитель
fc841ddc66
Коммит
5adb247914
|
|
@ -1,3 +1,10 @@
|
|||
Wed Aug 6 20:44:41 2008 Tanaka Akira <akr@fsij.org>
|
||||
|
||||
* tool/transcode-tblgen.rb: distinguish UNDEF and INVALID.
|
||||
[ruby-dev:35709]
|
||||
|
||||
* transcode.c (transcode_loop): don't need rb_enc_mbclen now.
|
||||
|
||||
Wed Aug 6 14:40:11 2008 Nobuyoshi Nakada <nobu@ruby-lang.org>
|
||||
|
||||
* common.mk (transdb.h): requires transcoders.
|
||||
|
|
|
|||
|
|
@ -17,6 +17,9 @@ end
|
|||
|
||||
class StrSet
|
||||
def self.parse(pattern)
|
||||
if /\A\s*(([0-9a-f][0-9a-f]|\{([0-9a-f][0-9a-f]|[0-9a-f][0-9a-f]-[0-9a-f][0-9a-f])(,([0-9a-f][0-9a-f]|[0-9a-f][0-9a-f]-[0-9a-f][0-9a-f]))*\})+(\s+|\z))*\z/i !~ pattern
|
||||
raise ArgumentError, "invalid pattern: #{pattern.inspect}"
|
||||
end
|
||||
result = []
|
||||
pattern.scan(/\S+/) {|seq|
|
||||
seq_result = []
|
||||
|
|
@ -69,25 +72,27 @@ class StrSet
|
|||
def to_s
|
||||
if @pat.empty?
|
||||
"(empset)"
|
||||
elsif @pat == [[]]
|
||||
"(empstr)"
|
||||
else
|
||||
@pat.map {|seq|
|
||||
seq.map {|byteset|
|
||||
if byteset.length == 1 && byteset[0].begin == byteset[0].end
|
||||
"%02x" % byteset[0].begin
|
||||
else
|
||||
"{" +
|
||||
byteset.map {|range|
|
||||
if range.begin == range.end
|
||||
"%02x" % range.begin
|
||||
else
|
||||
"%02x-%02x" % [range.begin, range.end]
|
||||
end
|
||||
}.join(',') +
|
||||
"}"
|
||||
end
|
||||
}.join('')
|
||||
if seq.empty?
|
||||
"(empstr)"
|
||||
else
|
||||
seq.map {|byteset|
|
||||
if byteset.length == 1 && byteset[0].begin == byteset[0].end
|
||||
"%02x" % byteset[0].begin
|
||||
else
|
||||
"{" +
|
||||
byteset.map {|range|
|
||||
if range.begin == range.end
|
||||
"%02x" % range.begin
|
||||
else
|
||||
"%02x-%02x" % [range.begin, range.end]
|
||||
end
|
||||
}.join(',') +
|
||||
"}"
|
||||
end
|
||||
}.join('')
|
||||
end
|
||||
}.join(' ')
|
||||
end
|
||||
end
|
||||
|
|
@ -142,9 +147,7 @@ class ActionMap
|
|||
|
||||
def initialize(h)
|
||||
@map = h
|
||||
@default_action = :undef
|
||||
end
|
||||
attr_accessor :default_action
|
||||
|
||||
def hash
|
||||
hash = 0
|
||||
|
|
@ -174,7 +177,7 @@ class ActionMap
|
|||
nil
|
||||
end
|
||||
|
||||
def each_firstbyte
|
||||
def each_firstbyte(valid_encoding=nil)
|
||||
h = {}
|
||||
@map.each {|ss, action|
|
||||
if ss.emptyable?
|
||||
|
|
@ -184,17 +187,27 @@ class ActionMap
|
|||
h[byte] ||= {}
|
||||
if h[byte][rest]
|
||||
raise "ambiguous"
|
||||
else
|
||||
h[byte][rest] = action
|
||||
end
|
||||
h[byte][rest] = action
|
||||
}
|
||||
end
|
||||
}
|
||||
h.keys.sort.each {|byte|
|
||||
am = ActionMap.new(h[byte])
|
||||
am.default_action = @default_action
|
||||
yield byte, am
|
||||
}
|
||||
if valid_encoding
|
||||
valid_encoding.each_firstbyte {|byte, rest|
|
||||
if h[byte]
|
||||
am = ActionMap.new(h[byte])
|
||||
yield byte, am, rest
|
||||
else
|
||||
am = ActionMap.new(rest => :undef)
|
||||
yield byte, am, nil
|
||||
end
|
||||
}
|
||||
else
|
||||
h.keys.sort.each {|byte|
|
||||
am = ActionMap.new(h[byte])
|
||||
yield byte, am, nil
|
||||
}
|
||||
end
|
||||
end
|
||||
|
||||
OffsetsMemo = {}
|
||||
|
|
@ -257,24 +270,14 @@ class ActionMap
|
|||
offsets = []
|
||||
infos = []
|
||||
infomap = {}
|
||||
noaction_bytes = []
|
||||
table.each_with_index {|action, byte|
|
||||
if !action
|
||||
noaction_bytes << byte
|
||||
next
|
||||
end
|
||||
action ||= :invalid
|
||||
unless o = infomap[action]
|
||||
infomap[action] = o = infos.length
|
||||
infos[o] = action
|
||||
end
|
||||
offsets[byte] = o
|
||||
}
|
||||
if !noaction_bytes.empty?
|
||||
noaction_bytes.each {|byte|
|
||||
offsets[byte] = infos.length
|
||||
}
|
||||
infos << @default_action
|
||||
end
|
||||
|
||||
if n = OffsetsMemo[offsets]
|
||||
offsets_name = n
|
||||
|
|
@ -315,15 +318,15 @@ End
|
|||
PostMemo = {}
|
||||
NextName = "a"
|
||||
|
||||
def generate_node(code, name_hint=nil, ranges=[])
|
||||
def generate_node(code, name_hint=nil, ranges=[], valid_encoding=nil)
|
||||
ranges = [0x00..0xff] if ranges.empty?
|
||||
range = ranges.first
|
||||
if n = PreMemo[self]
|
||||
if n = PreMemo[[self,valid_encoding]]
|
||||
return n
|
||||
end
|
||||
|
||||
table = Array.new(range.end - range.begin + 1)
|
||||
each_firstbyte {|byte, rest|
|
||||
each_firstbyte(valid_encoding) {|byte, rest, rest_valid_encoding|
|
||||
unless range === byte
|
||||
raise "byte not in range"
|
||||
end
|
||||
|
|
@ -332,7 +335,7 @@ End
|
|||
else
|
||||
name_hint2 = nil
|
||||
name_hint2 = "#{name_hint}_#{'%02X' % byte}" if name_hint
|
||||
table[byte-range.begin] = "&" + rest.generate_node(code, name_hint2, ranges[1..-1])
|
||||
table[byte-range.begin] = "&" + rest.generate_node(code, name_hint2, ranges[1..-1], rest_valid_encoding)
|
||||
end
|
||||
}
|
||||
|
||||
|
|
@ -345,7 +348,7 @@ End
|
|||
NextName.succ!
|
||||
end
|
||||
|
||||
PreMemo[self] = PostMemo[table] = name_hint
|
||||
PreMemo[[self,valid_encoding]] = PostMemo[table] = name_hint
|
||||
|
||||
code << generate_lookup_node(name_hint, table)
|
||||
name_hint
|
||||
|
|
@ -371,9 +374,15 @@ def transcode_compile_tree(name, from, map)
|
|||
}
|
||||
am = ActionMap.parse(h)
|
||||
|
||||
if ValidEncoding[from]
|
||||
valid_encoding = StrSet.parse(ValidEncoding[from])
|
||||
else
|
||||
valid_encoding = nil
|
||||
end
|
||||
|
||||
ranges = from == "UTF-8" ? [0x00..0xff, 0x80..0xbf, 0x80..0xbf, 0x80..0xbf] : []
|
||||
code = ''
|
||||
defined_name = am.generate_node(code, name, ranges)
|
||||
defined_name = am.generate_node(code, name, ranges, valid_encoding)
|
||||
return defined_name, code
|
||||
end
|
||||
|
||||
|
|
@ -419,75 +428,72 @@ def transcode_register_code
|
|||
code
|
||||
end
|
||||
|
||||
Universe = {
|
||||
"singlebyte" => "{00-ff}",
|
||||
"doublebyte" => "{00-ff}{00-ff}",
|
||||
"quadruplebyte" => "{00-ff}{00-ff}{00-ff}{00-ff}",
|
||||
"US-ASCII" => "{00-7f}",
|
||||
"EUC-JP" => <<-End,
|
||||
{00-7f}
|
||||
{a1-fe}{a1-fe}
|
||||
8e{a1-fe}
|
||||
8f{a1-fe}{a1-fe}
|
||||
End
|
||||
"EUC-KR" => <<-End,
|
||||
{00-7f}
|
||||
{a1-fe}{a1-fe}
|
||||
End
|
||||
"EUC-TW" => <<-End,
|
||||
{00-7f}
|
||||
{a1-fe}{a1-fe}
|
||||
8e{a1-b0}{a1-fe}{a1-fe}
|
||||
End
|
||||
"Shift_JIS" => <<-End,
|
||||
{00-7f}
|
||||
{81-9f,e0-fc}{40-7e,80-fc}
|
||||
{a1-df}
|
||||
End
|
||||
"Big5" => <<-End,
|
||||
{00-7f}
|
||||
{a1-fe}{40-7e,a1-fe}
|
||||
End
|
||||
"GBK" => <<-End,
|
||||
{00-80}
|
||||
{81-fe}{40-7e,80-fe}
|
||||
End
|
||||
"CP949" => <<-End,
|
||||
{00-80}
|
||||
{81-fe}{41-5a,61-7a,81-fe}
|
||||
End
|
||||
"UTF-8" => <<-End,
|
||||
{00-7f}
|
||||
{c2-df}{80-bf}
|
||||
e0{a0-bf}{80-bf}
|
||||
{e1-ec}{80-bf}{80-bf}
|
||||
ed{80-9f}{80-bf}
|
||||
{ee-ef}{80-bf}{80-bf}
|
||||
f0{90-bf}{80-bf}{80-bf}
|
||||
{f1-f3}{80-bf}{80-bf}{80-bf}
|
||||
f4{80-8f}{80-bf}{80-bf}
|
||||
End
|
||||
"GB18030" => <<-End,
|
||||
{00-7f}
|
||||
{81-fe}{40-7e,80-fe}
|
||||
{81-fe}{30-93}{81-fe}{30-93}
|
||||
End
|
||||
"UTF-16BE" => <<-End,
|
||||
{00-d7,e0-ff}{00-ff}
|
||||
{d8-db}{00-ff}{dc-df}{00-ff}
|
||||
End
|
||||
"UTF-16LE" => <<-End,
|
||||
{00-ff}{00-d7,e0-ff}
|
||||
{00-ff}{d8-db}{00-ff}{dc-df}
|
||||
End
|
||||
"UTF-32BE" => <<-End,
|
||||
0000{00-d7,e0-ff}{00-ff}
|
||||
00{01-10}{00-ff}{00-ff}
|
||||
End
|
||||
"UTF-32LE" => <<-End,
|
||||
{00-ff}{00-d7,e0-ff}0000
|
||||
{00-ff}{00-ff}{01-10}00
|
||||
End
|
||||
ValidEncoding = {
|
||||
'1byte' => '{00-ff}',
|
||||
'2byte' => '{00-ff}{00-ff}',
|
||||
'4byte' => '{00-ff}{00-ff}{00-ff}{00-ff}',
|
||||
'US-ASCII' => '{00-7f}',
|
||||
'UTF-8' => '{00-7f}
|
||||
{c2-df}{80-bf}
|
||||
e0{a0-bf}{80-bf}
|
||||
{e1-ec}{80-bf}{80-bf}
|
||||
ed{80-9f}{80-bf}
|
||||
{ee-ef}{80-bf}{80-bf}
|
||||
f0{90-bf}{80-bf}{80-bf}
|
||||
{f1-f3}{80-bf}{80-bf}{80-bf}
|
||||
f4{80-8f}{80-bf}{80-bf}',
|
||||
'UTF-16BE' => '{00-d7,e0-ff}{00-ff}
|
||||
{d8-db}{00-ff}{dc-df}{00-ff}',
|
||||
'UTF-16LE' => '{00-ff}{00-d7,e0-ff}
|
||||
{00-ff}{d8-db}{00-ff}{dc-df}',
|
||||
'UTF-32BE' => '0000{00-d7,e0-ff}{00-ff}
|
||||
00{01-10}{00-ff}{00-ff}',
|
||||
'UTF-32LE' => '{00-ff}{00-d7,e0-ff}0000
|
||||
{00-ff}{00-ff}{01-10}00',
|
||||
'EUC-JP' => '{00-7f}
|
||||
{a1-fe}{a1-fe}
|
||||
8e{a1-fe}
|
||||
8f{a1-fe}{a1-fe}',
|
||||
'CP51932' => '{00-7f}
|
||||
{a1-fe}{a1-fe}
|
||||
8e{a1-fe}',
|
||||
'Shift_JIS' => '{00-7f}
|
||||
{81-9f,e0-fc}{40-7e,80-fc}
|
||||
{a1-df}',
|
||||
'EUC-KR' => '{00-7f}
|
||||
{a1-fe}{a1-fe}',
|
||||
'CP949' => '{00-7f}
|
||||
{81-fe}{41-5a,61-7a,81-fe}',
|
||||
'Big5' => '{00-7f}
|
||||
{81-fe}{40-7e,a1-fe}',
|
||||
'EUC-TW' => '{00-7f}
|
||||
{a1-fe}{a1-fe}
|
||||
8e{a1-b0}{a1-fe}{a1-fe}',
|
||||
'GBK' => '{00-80}
|
||||
{81-fe}{40-7e,80-fe}',
|
||||
'GB18030' => '{00-7f}
|
||||
{81-fe}{40-7e,80-fe}
|
||||
{81-fe}{30-39}{81-fe}{30-39}',
|
||||
}
|
||||
|
||||
{
|
||||
'ISO-8859-1' => '1byte',
|
||||
'ISO-8859-2' => '1byte',
|
||||
'ISO-8859-3' => '1byte',
|
||||
'ISO-8859-4' => '1byte',
|
||||
'ISO-8859-5' => '1byte',
|
||||
'ISO-8859-6' => '1byte',
|
||||
'ISO-8859-7' => '1byte',
|
||||
'ISO-8859-8' => '1byte',
|
||||
'ISO-8859-9' => '1byte',
|
||||
'ISO-8859-10' => '1byte',
|
||||
'ISO-8859-11' => '1byte',
|
||||
'ISO-8859-13' => '1byte',
|
||||
'ISO-8859-14' => '1byte',
|
||||
'ISO-8859-15' => '1byte',
|
||||
'Windows-31J' => 'Shift_JIS',
|
||||
}.each {|k, v|
|
||||
ValidEncoding[k] = ValidEncoding.fetch(v)
|
||||
}
|
||||
|
||||
def make_signature(filename, src)
|
||||
|
|
@ -528,7 +534,7 @@ if !force_mode && output_filename && File.readable?(output_filename)
|
|||
end
|
||||
end
|
||||
}
|
||||
if old_signature == chk_signature
|
||||
if old_signature == chk_signature && File.mtime(__FILE__) < File.mtime(output_filename)
|
||||
now = Time.now
|
||||
File.utime(now, now, output_filename)
|
||||
STDERR.puts "already up-to-date: #{output_filename}" if VERBOSE_MODE
|
||||
|
|
|
|||
|
|
@ -188,7 +188,6 @@ transcode_loop(const unsigned char **in_pos, unsigned char **out_pos,
|
|||
unsigned char next_byte;
|
||||
int from_utf8 = my_transcoder->from_utf8;
|
||||
unsigned char *out_s = out_stop - my_transcoder->max_output + 1;
|
||||
rb_encoding *from_encoding = rb_enc_find(my_transcoder->from_encoding);
|
||||
rb_encoding *to_encoding = rb_enc_find(my_transcoder->to_encoding);
|
||||
|
||||
while (in_p < in_stop) {
|
||||
|
|
@ -280,10 +279,6 @@ transcode_loop(const unsigned char **in_pos, unsigned char **out_pos,
|
|||
/* valid character in from encoding
|
||||
* but no related character(s) in to encoding */
|
||||
/* todo: add more alternative behaviors */
|
||||
{
|
||||
int len = rb_enc_mbclen((const char *)char_start, (const char *)in_stop, from_encoding);
|
||||
while (in_p < char_start + len) in_p++;
|
||||
}
|
||||
if (opt&UNDEF_IGNORE) {
|
||||
continue;
|
||||
}
|
||||
|
|
|
|||
Загрузка…
Ссылка в новой задаче