зеркало из https://github.com/github/ruby.git
* transcode_data.h, transcode.c, tool/transcode-tblgen.rb: Added
support for new transcoding instruction FUNsio (with Tatsuya Mizuno) * enc/trans/gb18030.trans: Significantly reduced GB18030 conversion table footprint using FUNsio and differences (with Tatsuya Mizuno) * test/ruby/test_transcode.rb: Minor name fix (from Tatsuya Mizuno) git-svn-id: svn+ssh://ci.ruby-lang.org/ruby/trunk@26065 b2dd03c8-39d4-4d8f-98ff-823fe69b080e
This commit is contained in:
duerst
Родитель
136c25ebf0
Коммит
b32ee85f97
10
ChangeLog
10
ChangeLog
|
|
@ -1,3 +1,13 @@
|
|||
Thu Dec 10 20:55:55 2009 Martin Duerst <duerst@it.aoyama.ac.jp>
|
||||
|
||||
* transcode_data.h, transcode.c, tool/transcode-tblgen.rb: Added
|
||||
support for new transcoding instruction FUNsio (with Tatsuya Mizuno)
|
||||
|
||||
* enc/trans/gb18030.trans: Significantly reduced GB18030 conversion
|
||||
table footprint using FUNsio and differences (with Tatsuya Mizuno)
|
||||
|
||||
* test/ruby/test_transcode.rb: Minor name fix (from Tatsuya Mizuno)
|
||||
|
||||
Thu Dec 10 17:22:36 2009 Shugo Maeda <shugo@ruby-lang.org>
|
||||
|
||||
* vm_eval.c (yield_under): yields self the same as 1.8.
|
||||
|
|
|
|||
|
|
@ -3,15 +3,63 @@
|
|||
<%
|
||||
require "gb18030-tbl"
|
||||
|
||||
def linear(code)
|
||||
bytes = [code].pack('H8').unpack 'C4'
|
||||
((bytes[0]*10+bytes[1])*126+bytes[2])*10+bytes[3]
|
||||
end
|
||||
|
||||
def calculate_differences_gb_utf(table)
|
||||
table.collect do |code|
|
||||
code = code.dup
|
||||
if code[0].length == 4
|
||||
if code[1] < 0x800 # GB-18030: 2 bytes, UTF-8: 2 bytes
|
||||
# do nothing
|
||||
else # GB-18030: 2 bytes, UTF-8: 3 bytes
|
||||
gb_linear2b = code[0].to_i(16)
|
||||
diff2b = gb_linear2b - code[1] + 24055
|
||||
code[1] = "funsio(#{diff2b})"
|
||||
end
|
||||
else
|
||||
if code[1] < 0x800 # GB-18030: 4 bytes, UTF-8: 2 bytes
|
||||
# do nothing
|
||||
else # GB-18030: 4 bytes, UTF-8: 3 bytes
|
||||
gb_linear4b = linear(code[0])
|
||||
diff4b = gb_linear4b - code[1] - 0x170000
|
||||
code[1] = "funsio(#{diff4b})"
|
||||
end
|
||||
end
|
||||
code
|
||||
end
|
||||
end
|
||||
|
||||
def calculate_differences_utf_gb(table)
|
||||
table_rev = table.map{|a,b| [b,a]}
|
||||
table_rev.collect do |code|
|
||||
code = code.dup
|
||||
if code[0] >= 0x800
|
||||
if code[1].length == 4 #GB18030 2byte UTF-8 3byte
|
||||
gb_linear2b = code[1].to_i(16)
|
||||
diff2b = gb_linear2b - code[0] + 24055
|
||||
code[1] = "funsio(#{diff2b})"
|
||||
else # GB-18030: 4 bytes, UTF-8: 3 bytes
|
||||
gb_linear4b = linear(code[1])
|
||||
diff4b = gb_linear4b - code[0] - 0x170000
|
||||
code[1] = "funsio(#{diff4b})"
|
||||
end
|
||||
end
|
||||
code
|
||||
end
|
||||
end
|
||||
|
||||
transcode_tbl_only "GB18030", "UTF-8", [["{00-7f}", :nomap]] +
|
||||
GB18030_TO_UCS_TBL + [
|
||||
calculate_differences_gb_utf(GB18030_TO_UCS_TBL) + [
|
||||
["{90-e2}{30-39}{81-fe}{30-39}", :func_so],
|
||||
["e3{30-31}{81-fe}{30-39}", :func_so],
|
||||
["e332{81-99}{30-39}", :func_so],
|
||||
["e3329a{30-35}", :func_so], # "E3329A35" is U+10FFFF
|
||||
]
|
||||
transcode_tbl_only "UTF-8", "GB18030", [["{00-7f}", :nomap]] +
|
||||
GB18030_TO_UCS_TBL.map {|a,b| [b,a] } + [
|
||||
calculate_differences_utf_gb(GB18030_TO_UCS_TBL) + [
|
||||
["f0{90-bf}{80-bf}{80-bf}", :func_so],
|
||||
["{f1-f3}{80-bf}{80-bf}{80-bf}", :func_so],
|
||||
["f4{80-8f}{80-bf}{80-bf}", :func_so]
|
||||
|
|
@ -20,6 +68,7 @@
|
|||
|
||||
<%= transcode_generated_code %>
|
||||
|
||||
/* GB18030 4byte, UTF-8 4byte*/
|
||||
static ssize_t
|
||||
fun_so_from_gb18030(void *statep, const unsigned char *s, size_t l, unsigned char *o, size_t osize)
|
||||
{
|
||||
|
|
@ -33,6 +82,7 @@ fun_so_from_gb18030(void *statep, const unsigned char *s, size_t l, unsigned cha
|
|||
return 4;
|
||||
}
|
||||
|
||||
/* GB18030 4byte, UTF-8 4byte*/
|
||||
static ssize_t
|
||||
fun_so_to_gb18030(void *statep, const unsigned char *s, size_t l, unsigned char *o, size_t osize)
|
||||
{
|
||||
|
|
@ -49,6 +99,56 @@ fun_so_to_gb18030(void *statep, const unsigned char *s, size_t l, unsigned char
|
|||
return 4;
|
||||
}
|
||||
|
||||
/* GB18030 2byte, UTF-8 3byte and GB18030 4byte, UTF-8 3byte*/
|
||||
static ssize_t
|
||||
fun_sio_from_gb18030(void *statep, const unsigned char *s, size_t l, VALUE info, unsigned char *o, size_t osize)
|
||||
{
|
||||
unsigned int diff = (unsigned int)(info >> 8);
|
||||
unsigned int u; /* Unicode Scalar Value */
|
||||
if (diff & 0x20000) { /* GB18030 4 bytes */
|
||||
u = ((s[0]*10+s[1])*126+s[2])*10+s[3] - diff - 0x170000;
|
||||
}
|
||||
else { /* GB18030 2 bytes */
|
||||
u = s[0]*256 + s[1] + 24055 - diff;
|
||||
}
|
||||
o[0] = 0xE0 | (u>>12);
|
||||
o[1] = 0x80 | ((u>>6)&0x3F);
|
||||
o[2] = 0x80 | (u&0x3F);
|
||||
return 3;
|
||||
}
|
||||
|
||||
/* GB18030 2byte, UTF-8 3byte and GB18030 4byte, UTF-8 3byte*/
|
||||
static ssize_t
|
||||
fun_sio_to_gb18030(void *statep, const unsigned char *s, size_t l, VALUE info, unsigned char *o, size_t osize)
|
||||
{
|
||||
unsigned int diff = (unsigned int)(info >> 8);
|
||||
unsigned int u; /* Unicode Scalar Value */
|
||||
|
||||
u = ((s[0]&0x0F)<<12) | ((s[1]&0x3F)<<6) | (s[2]&0x3F);
|
||||
|
||||
if (diff & 0x20000) { /* GB18030 4 bytes */
|
||||
u += (diff + 0x170000);
|
||||
u -= 1688980;
|
||||
u += 0x2;
|
||||
o[3] = 0x30 + u%10;
|
||||
u /= 10;
|
||||
u += 0x32;
|
||||
o[2] = 0x81 + u%126;
|
||||
u /= 126;
|
||||
u += 0x1;
|
||||
o[1] = 0x30 + u%10;
|
||||
u /= 10;
|
||||
o[0] = 0x81 + u;
|
||||
return 4;
|
||||
}
|
||||
else { /* GB18030 2 bytes */
|
||||
u += (diff - 24055);
|
||||
o[1] = u%256;
|
||||
o[0] = u/256;
|
||||
return 2;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
static const rb_transcoder
|
||||
rb_from_GB18030 = {
|
||||
|
|
@ -56,11 +156,11 @@ rb_from_GB18030 = {
|
|||
TRANSCODE_TABLE_INFO,
|
||||
1, /* input_unit_length */
|
||||
4, /* max_input */
|
||||
3, /* max_output */
|
||||
4, /* max_output */
|
||||
asciicompat_converter, /* asciicompat_type */
|
||||
0, NULL, NULL, /* state_size, state_init, state_fini */
|
||||
NULL, NULL, NULL, fun_so_from_gb18030,
|
||||
NULL, NULL, NULL
|
||||
NULL, NULL, NULL, fun_sio_from_gb18030
|
||||
};
|
||||
static const rb_transcoder
|
||||
rb_to_GB18030 = {
|
||||
|
|
@ -72,7 +172,7 @@ rb_to_GB18030 = {
|
|||
asciicompat_converter, /* asciicompat_type */
|
||||
0, NULL, NULL, /* state_size, state_init, state_fini */
|
||||
NULL, NULL, NULL, fun_so_to_gb18030,
|
||||
NULL, NULL, NULL
|
||||
NULL, NULL, NULL, fun_sio_to_gb18030
|
||||
};
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -1579,10 +1579,10 @@ class TestTranscode < Test::Unit::TestCase
|
|||
|
||||
def test_gb18030
|
||||
# overall roundtrip test
|
||||
all_gb18030 = (0x0..0xD7FF).to_a.pack 'U*' #追加
|
||||
all_gb18030 << (0xE000..0xFFFF).to_a.pack("U*") #追加
|
||||
all_unicode = (0x0..0xD7FF).to_a.pack 'U*' #追加
|
||||
all_unicode << (0xE000..0xFFFF).to_a.pack("U*") #追加
|
||||
|
||||
assert_equal(all_gb18030, all_gb18030.encode("gb18030").encode("UTF-8")) #追加
|
||||
assert_equal(all_unicode, all_unicode.encode("gb18030").encode("UTF-8")) #追加
|
||||
|
||||
# tests from GBK
|
||||
check_both_ways("\u4E02", "\x81\x40", 'GB18030') #
|
||||
|
|
|
|||
|
|
@ -339,6 +339,8 @@ class ActionMap
|
|||
"o2(0x#$1,0x#$2)"
|
||||
when /\A([0-9a-f][0-9a-f])([0-9a-f][0-9a-f])([0-9a-f][0-9a-f])\z/i
|
||||
"o3(0x#$1,0x#$2,0x#$3)"
|
||||
when /funsio\((\d+)\)/
|
||||
"funsio(#{$1})"
|
||||
when /\A([0-9a-f][0-9a-f])(3[0-9])([0-9a-f][0-9a-f])(3[0-9])\z/i
|
||||
"g4(0x#$1,0x#$2,0x#$3,0x#$4)"
|
||||
when /\A(f[0-7])([0-9a-f][0-9a-f])([0-9a-f][0-9a-f])([0-9a-f][0-9a-f])\z/i
|
||||
|
|
|
|||
26
transcode.c
26
transcode.c
|
|
@ -505,6 +505,8 @@ transcode_restartable0(const unsigned char **in_pos, unsigned char **out_pos,
|
|||
case 30: goto resume_label30;
|
||||
case 31: goto resume_label31;
|
||||
case 32: goto resume_label32;
|
||||
case 33: goto resume_label33;
|
||||
case 34: goto resume_label34;
|
||||
}
|
||||
|
||||
while (1) {
|
||||
|
|
@ -649,6 +651,30 @@ transcode_restartable0(const unsigned char **in_pos, unsigned char **out_pos,
|
|||
}
|
||||
break;
|
||||
}
|
||||
case FUNsio:
|
||||
{
|
||||
const unsigned char *char_start;
|
||||
size_t char_len;
|
||||
SUSPEND_OBUF(33);
|
||||
if (tr->max_output <= out_stop - out_p) {
|
||||
char_start = transcode_char_start(tc, *in_pos, inchar_start, in_p, &char_len);
|
||||
out_p += tr->func_sio(TRANSCODING_STATE(tc),
|
||||
char_start, (size_t)char_len, next_info,
|
||||
out_p, out_stop - out_p);
|
||||
}
|
||||
else {
|
||||
char_start = transcode_char_start(tc, *in_pos, inchar_start, in_p, &char_len);
|
||||
writebuf_len = tr->func_sio(TRANSCODING_STATE(tc),
|
||||
char_start, (size_t)char_len, next_info,
|
||||
TRANSCODING_WRITEBUF(tc), TRANSCODING_WRITEBUF_SIZE(tc));
|
||||
writebuf_off = 0;
|
||||
while (writebuf_off < writebuf_len) {
|
||||
SUSPEND_OBUF(34);
|
||||
*out_p++ = TRANSCODING_WRITEBUF(tc)[writebuf_off++];
|
||||
}
|
||||
}
|
||||
break;
|
||||
}
|
||||
case INVALID:
|
||||
if (tc->recognized_len + (in_p - inchar_start) <= unitlen) {
|
||||
if (tc->recognized_len + (in_p - inchar_start) < unitlen)
|
||||
|
|
|
|||
|
|
@ -36,6 +36,7 @@
|
|||
#define FUNso (PType 0x0F) /* function from start to output */
|
||||
#define STR1 (PType 0x11) /* string 4 <= len <= 259 bytes: 1byte length + content */
|
||||
#define GB4bt (PType 0x12) /* GB18030 four bytes payload */
|
||||
#define FUNsio (PType 0x13) /* function from start and info to output */
|
||||
|
||||
#define STR1_LENGTH(byte_addr) (unsigned int)(*(byte_addr) + 4)
|
||||
#define STR1_BYTEINDEX(w) ((w) >> 6)
|
||||
|
|
@ -47,6 +48,7 @@
|
|||
#define o3(b1,b2,b3) (PType(((((unsigned char)(b1))<<8)|(((unsigned char)(b2))<<16)|(((unsigned int)(unsigned char)(b3))<<24)|THREEbt)&0xffffffffU))
|
||||
#define o4(b0,b1,b2,b3) (PType(((((unsigned char)(b1))<<8)|(((unsigned char)(b2))<<16)|(((unsigned char)(b3))<<24)|((((unsigned char)(b0))&0x07)<<5)|FOURbt)&0xffffffffU))
|
||||
#define g4(b0,b1,b2,b3) (PType(((((unsigned char)(b0))<<8)|(((unsigned char)(b2))<<16)|((((unsigned char)(b1))&0x0f)<<24)|((((unsigned int)(unsigned char)(b3))&0x0f)<<28)|GB4bt)&0xffffffffU))
|
||||
#define funsio(diff) (PType((((unsigned int)(diff))<<8)|FUNsio))
|
||||
|
||||
#define getBT1(a) ((unsigned char)((a)>> 8))
|
||||
#define getBT2(a) ((unsigned char)((a)>>16))
|
||||
|
|
@ -98,6 +100,7 @@ struct rb_transcoder {
|
|||
ssize_t (*finish_func)(void*, unsigned char*, size_t); /* -> output */
|
||||
ssize_t (*resetsize_func)(void*); /* -> len */
|
||||
ssize_t (*resetstate_func)(void*, unsigned char*, size_t); /* -> output */
|
||||
ssize_t (*func_sio)(void*, const unsigned char*, size_t, VALUE, unsigned char*, size_t); /* start -> output */
|
||||
};
|
||||
|
||||
void rb_declare_transcoder(const char *enc1, const char *enc2, const char *lib);
|
||||
|
|
|
|||
Загрузка…
Ссылка в новой задаче