зеркало из https://github.com/mozilla/gecko-dev.git
Bug 1402271 - patch 1 - Simplify the genUnicodePropertyData.pl tool to generate only the data we need to supplement ICU properties. r=m_kato
This commit is contained in:
Jonathan Kew
Родитель
42bd74780c
Коммит
26d09a9bd2
|
|
@ -9,6 +9,10 @@
|
|||
# read from the Unicode Character Database and compiled into multi-level arrays
|
||||
# for efficient lookup.
|
||||
#
|
||||
# Note that for most properties, we now rely on ICU; this tool and the tables
|
||||
# it generates are used only for a couple of properties not readily exposed
|
||||
# via ICU APIs.
|
||||
#
|
||||
# To regenerate the tables in nsUnicodePropertyData.cpp:
|
||||
#
|
||||
# (1) Download the current Unicode data files from
|
||||
|
|
@ -17,13 +21,6 @@
|
|||
#
|
||||
# NB: not all the files are actually needed; currently, we require
|
||||
# - UnicodeData.txt
|
||||
# - Scripts.txt
|
||||
# - BidiMirroring.txt
|
||||
# - BidiBrackets.txt
|
||||
# - HangulSyllableType.txt
|
||||
# - LineBreak.txt
|
||||
# - EastAsianWidth.txt
|
||||
# - DerivedCoreProperties.txt
|
||||
# - ReadMe.txt (to record version/date of the UCD)
|
||||
# - Unihan_Variants.txt (from Unihan.zip)
|
||||
# though this may change if we find a need for additional properties.
|
||||
|
|
@ -44,7 +41,6 @@
|
|||
# (2) Run this tool using a command line of the form
|
||||
#
|
||||
# perl genUnicodePropertyData.pl \
|
||||
# /path/to/harfbuzz/src \
|
||||
# /path/to/icu/common/unicode \
|
||||
# /path/to/UCD-directory
|
||||
#
|
||||
|
|
@ -58,17 +54,15 @@
|
|||
use strict;
|
||||
use List::Util qw(first);
|
||||
|
||||
if ($#ARGV != 2) {
|
||||
if ($#ARGV != 1) {
|
||||
print <<__EOT;
|
||||
# Run this tool using a command line of the form
|
||||
#
|
||||
# perl genUnicodePropertyData.pl \\
|
||||
# /path/to/harfbuzz/src \\
|
||||
# /path/to/icu/common/unicode \\
|
||||
# /path/to/UCD-directory
|
||||
#
|
||||
# where harfbuzz/src is the directory containing harfbuzz .cc and .hh files,
|
||||
# icu/common/unicode is the directory containing ICU 'common' public headers,
|
||||
# where icu/common/unicode is the directory containing ICU 'common' headers,
|
||||
# and UCD-directory is a directory containing the current Unicode Character
|
||||
# Database files (UnicodeData.txt, etc), available from
|
||||
# http://www.unicode.org/Public/UNIDATA/, with additional resources as
|
||||
|
|
@ -84,35 +78,11 @@ __EOT
|
|||
exit 0;
|
||||
}
|
||||
|
||||
my $HARFBUZZ = $ARGV[0];
|
||||
my $ICU = $ARGV[1];
|
||||
my $UNICODE = $ARGV[2];
|
||||
my $ICU = $ARGV[0];
|
||||
my $UNICODE = $ARGV[1];
|
||||
|
||||
# load HB_Category constants
|
||||
|
||||
my $cc = -1;
|
||||
my %catCode;
|
||||
|
||||
sub readHarfBuzzHeader
|
||||
{
|
||||
my $file = shift;
|
||||
open FH, "< $HARFBUZZ/$file" or die "can't open harfbuzz header $HARFBUZZ/$file\n";
|
||||
while (<FH>) {
|
||||
if (m/HB_UNICODE_GENERAL_CATEGORY_([A-Z_]+)/) {
|
||||
$cc++;
|
||||
$catCode{$1} = $cc;
|
||||
}
|
||||
}
|
||||
close FH;
|
||||
}
|
||||
|
||||
&readHarfBuzzHeader("hb-unicode.h");
|
||||
|
||||
die "didn't find HarfBuzz category codes\n" if $cc == -1;
|
||||
|
||||
my %scriptCode;
|
||||
my @scriptCodeToTag;
|
||||
my @scriptCodeToName;
|
||||
my @idtype;
|
||||
|
||||
my $sc = -1;
|
||||
|
||||
|
|
@ -129,8 +99,6 @@ sub readIcuHeader
|
|||
s/SIGN_WRITING/SIGNWRITING/;
|
||||
if (m|USCRIPT_([A-Z_]+)\s*=\s*([0-9]+),\s*/\*\s*([A-Z][a-z]{3})\s*\*/|) {
|
||||
$sc = $2;
|
||||
$scriptCode{$1} = $sc;
|
||||
$scriptCodeToTag[$sc] = $3;
|
||||
$scriptCodeToName[$sc] = $1;
|
||||
}
|
||||
}
|
||||
|
|
@ -170,32 +138,6 @@ my %mappedIdType = (
|
|||
"Allowed" => 1
|
||||
);
|
||||
|
||||
my %bidicategoryCode = (
|
||||
"L" => 0, # Left-to-Right
|
||||
"R" => 1, # Right-to-Left
|
||||
"EN" => 2, # European Number
|
||||
"ES" => 3, # European Number Separator
|
||||
"ET" => 4, # European Number Terminator
|
||||
"AN" => 5, # Arabic Number
|
||||
"CS" => 6, # Common Number Separator
|
||||
"B" => 7, # Paragraph Separator
|
||||
"S" => 8, # Segment Separator
|
||||
"WS" => 9, # Whitespace
|
||||
"ON" => 10, # Other Neutrals
|
||||
"LRE" => 11, # Left-to-Right Embedding
|
||||
"LRO" => 12, # Left-to-Right Override
|
||||
"AL" => 13, # Right-to-Left Arabic
|
||||
"RLE" => 14, # Right-to-Left Embedding
|
||||
"RLO" => 15, # Right-to-Left Override
|
||||
"PDF" => 16, # Pop Directional Format
|
||||
"NSM" => 17, # Non-Spacing Mark
|
||||
"BN" => 18, # Boundary Neutral
|
||||
"FSI" => 19, # First Strong Isolate
|
||||
"LRI" => 20, # Left-to-Right Isolate
|
||||
"RLI" => 21, # Right-to-left Isolate
|
||||
"PDI" => 22 # Pop Direcitonal Isolate
|
||||
);
|
||||
|
||||
my %verticalOrientationCode = (
|
||||
'U' => 0, # U - Upright, the same orientation as in the code charts
|
||||
'R' => 1, # R - Rotated 90 degrees clockwise compared to the code charts
|
||||
|
|
@ -203,141 +145,18 @@ my %verticalOrientationCode = (
|
|||
'Tr' => 3 # Tr - Transformed typographically, with fallback to Rotated
|
||||
);
|
||||
|
||||
my %lineBreakCode = ( # ordering matches ICU's ULineBreak enum
|
||||
"XX" => 0,
|
||||
"AI" => 1,
|
||||
"AL" => 2,
|
||||
"B2" => 3,
|
||||
"BA" => 4,
|
||||
"BB" => 5,
|
||||
"BK" => 6,
|
||||
"CB" => 7,
|
||||
"CL" => 8,
|
||||
"CM" => 9,
|
||||
"CR" => 10,
|
||||
"EX" => 11,
|
||||
"GL" => 12,
|
||||
"HY" => 13,
|
||||
"ID" => 14,
|
||||
"IN" => 15,
|
||||
"IS" => 16,
|
||||
"LF" => 17,
|
||||
"NS" => 18,
|
||||
"NU" => 19,
|
||||
"OP" => 20,
|
||||
"PO" => 21,
|
||||
"PR" => 22,
|
||||
"QU" => 23,
|
||||
"SA" => 24,
|
||||
"SG" => 25,
|
||||
"SP" => 26,
|
||||
"SY" => 27,
|
||||
"ZW" => 28,
|
||||
"NL" => 29,
|
||||
"WJ" => 30,
|
||||
"H2" => 31,
|
||||
"H3" => 32,
|
||||
"JL" => 33,
|
||||
"JT" => 34,
|
||||
"JV" => 35,
|
||||
"CP" => 36,
|
||||
"CJ" => 37,
|
||||
"HL" => 38,
|
||||
"RI" => 39,
|
||||
"EB" => 40,
|
||||
"EM" => 41,
|
||||
"ZWJ" => 42
|
||||
);
|
||||
|
||||
my %eastAsianWidthCode = (
|
||||
"N" => 0,
|
||||
"A" => 1,
|
||||
"H" => 2,
|
||||
"W" => 3,
|
||||
"F" => 4,
|
||||
"Na" => 5
|
||||
);
|
||||
|
||||
# initialize default properties
|
||||
my @script;
|
||||
my @category;
|
||||
my @combining;
|
||||
my @mirror;
|
||||
my @pairedBracketType;
|
||||
my @hangul;
|
||||
my @casemap;
|
||||
my @idtype;
|
||||
my @numericvalue;
|
||||
my @hanVariant;
|
||||
my @bidicategory;
|
||||
my @fullWidth;
|
||||
my @fullWidthInverse;
|
||||
my @verticalOrientation;
|
||||
my @lineBreak;
|
||||
my @eastAsianWidthFWH;
|
||||
my @defaultIgnorable;
|
||||
for (my $i = 0; $i < 0x110000; ++$i) {
|
||||
$script[$i] = $scriptCode{"UNKNOWN"};
|
||||
$category[$i] = $catCode{"UNASSIGNED"};
|
||||
$combining[$i] = 0;
|
||||
$pairedBracketType[$i] = 0;
|
||||
$casemap[$i] = 0;
|
||||
$idtype[$i] = $mappedIdType{'Restricted'};
|
||||
$numericvalue[$i] = -1;
|
||||
$hanVariant[$i] = 0;
|
||||
$bidicategory[$i] = $bidicategoryCode{"L"};
|
||||
$fullWidth[$i] = 0;
|
||||
$fullWidthInverse[$i] = 0;
|
||||
$verticalOrientation[$i] = 1; # default for unlisted codepoints is 'R'
|
||||
$lineBreak[$i] = $lineBreakCode{"XX"};
|
||||
$eastAsianWidthFWH[$i] = 0;
|
||||
$defaultIgnorable[$i] = 0;
|
||||
}
|
||||
|
||||
# blocks where the default for bidi category is not L
|
||||
for my $i (0x0600..0x07BF, 0x08A0..0x08FF, 0xFB50..0xFDCF, 0xFDF0..0xFDFF, 0xFE70..0xFEFF, 0x1EE00..0x0001EEFF) {
|
||||
$bidicategory[$i] = $bidicategoryCode{"AL"};
|
||||
}
|
||||
for my $i (0x0590..0x05FF, 0x07C0..0x089F, 0xFB1D..0xFB4F, 0x00010800..0x00010FFF, 0x0001E800..0x0001EDFF, 0x0001EF00..0x0001EFFF) {
|
||||
$bidicategory[$i] = $bidicategoryCode{"R"};
|
||||
}
|
||||
for my $i (0x20A0..0x20CF) {
|
||||
$bidicategory[$i] = $bidicategoryCode{"ET"};
|
||||
}
|
||||
|
||||
my %ucd2hb = (
|
||||
'Cc' => 'CONTROL',
|
||||
'Cf' => 'FORMAT',
|
||||
'Cn' => 'UNASSIGNED',
|
||||
'Co' => 'PRIVATE_USE',
|
||||
'Cs' => 'SURROGATE',
|
||||
'Ll' => 'LOWERCASE_LETTER',
|
||||
'Lm' => 'MODIFIER_LETTER',
|
||||
'Lo' => 'OTHER_LETTER',
|
||||
'Lt' => 'TITLECASE_LETTER',
|
||||
'Lu' => 'UPPERCASE_LETTER',
|
||||
'Mc' => 'SPACING_MARK',
|
||||
'Me' => 'ENCLOSING_MARK',
|
||||
'Mn' => 'NON_SPACING_MARK',
|
||||
'Nd' => 'DECIMAL_NUMBER',
|
||||
'Nl' => 'LETTER_NUMBER',
|
||||
'No' => 'OTHER_NUMBER',
|
||||
'Pc' => 'CONNECT_PUNCTUATION',
|
||||
'Pd' => 'DASH_PUNCTUATION',
|
||||
'Pe' => 'CLOSE_PUNCTUATION',
|
||||
'Pf' => 'FINAL_PUNCTUATION',
|
||||
'Pi' => 'INITIAL_PUNCTUATION',
|
||||
'Po' => 'OTHER_PUNCTUATION',
|
||||
'Ps' => 'OPEN_PUNCTUATION',
|
||||
'Sc' => 'CURRENCY_SYMBOL',
|
||||
'Sk' => 'MODIFIER_SYMBOL',
|
||||
'Sm' => 'MATH_SYMBOL',
|
||||
'So' => 'OTHER_SYMBOL',
|
||||
'Zl' => 'LINE_SEPARATOR',
|
||||
'Zp' => 'PARAGRAPH_SEPARATOR',
|
||||
'Zs' => 'SPACE_SEPARATOR'
|
||||
);
|
||||
|
||||
# read ReadMe.txt
|
||||
my @versionInfo;
|
||||
open FH, "< $UNICODE/ReadMe.txt" or die "can't open Unicode ReadMe.txt file\n";
|
||||
|
|
@ -347,12 +166,6 @@ while (<FH>) {
|
|||
}
|
||||
close FH;
|
||||
|
||||
my $kTitleToUpper = 0x80000000;
|
||||
my $kUpperToLower = 0x40000000;
|
||||
my $kLowerToTitle = 0x20000000;
|
||||
my $kLowerToUpper = 0x10000000;
|
||||
my $kCaseMapCharMask = 0x001fffff;
|
||||
|
||||
# read UnicodeData.txt
|
||||
open FH, "< $UNICODE/UnicodeData.txt" or die "can't open UCD file UnicodeData.txt\n";
|
||||
while (<FH>) {
|
||||
|
|
@ -365,12 +178,6 @@ while (<FH>) {
|
|||
if ($fields[1] =~ /Last/) {
|
||||
my $last = hex "0x$fields[0]";
|
||||
do {
|
||||
$category[$first] = $catCode{$ucd2hb{$fields[2]}};
|
||||
$combining[$first] = $fields[3];
|
||||
$bidicategory[$first] = $bidicategoryCode{$fields[4]};
|
||||
unless (length($fields[7]) == 0) {
|
||||
$numericvalue[$first] = $fields[7];
|
||||
}
|
||||
if ($fields[1] =~ /CJK/) {
|
||||
@hanVariant[$first] = 3;
|
||||
}
|
||||
|
|
@ -381,33 +188,6 @@ while (<FH>) {
|
|||
}
|
||||
} else {
|
||||
my $usv = hex "0x$fields[0]";
|
||||
$category[$usv] = $catCode{$ucd2hb{$fields[2]}};
|
||||
$combining[$usv] = $fields[3];
|
||||
my $upper = hex $fields[12];
|
||||
my $lower = hex $fields[13];
|
||||
my $title = hex $fields[14];
|
||||
# we only store one mapping for each character,
|
||||
# but also record what kind of mapping it is
|
||||
if ($upper && $lower) {
|
||||
$casemap[$usv] |= $kTitleToUpper;
|
||||
$casemap[$usv] |= ($usv ^ $upper);
|
||||
}
|
||||
elsif ($lower) {
|
||||
$casemap[$usv] |= $kUpperToLower;
|
||||
$casemap[$usv] |= ($usv ^ $lower);
|
||||
}
|
||||
elsif ($title && ($title != $upper)) {
|
||||
$casemap[$usv] |= $kLowerToTitle;
|
||||
$casemap[$usv] |= ($usv ^ $title);
|
||||
}
|
||||
elsif ($upper) {
|
||||
$casemap[$usv] |= $kLowerToUpper;
|
||||
$casemap[$usv] |= ($usv ^ $upper);
|
||||
}
|
||||
$bidicategory[$usv] = $bidicategoryCode{$fields[4]};
|
||||
unless (length($fields[7]) == 0) {
|
||||
$numericvalue[$usv] = $fields[7];
|
||||
}
|
||||
if ($fields[1] =~ /CJK/) {
|
||||
@hanVariant[$usv] = 3;
|
||||
}
|
||||
|
|
@ -427,178 +207,6 @@ while (<FH>) {
|
|||
}
|
||||
close FH;
|
||||
|
||||
# read Scripts.txt
|
||||
open FH, "< $UNICODE/Scripts.txt" or die "can't open UCD file Scripts.txt\n";
|
||||
push @versionInfo, "";
|
||||
while (<FH>) {
|
||||
chomp;
|
||||
push @versionInfo, $_;
|
||||
last if /Date:/;
|
||||
}
|
||||
while (<FH>) {
|
||||
if (m/([0-9A-F]{4,6})(?:\.\.([0-9A-F]{4,6}))*\s+;\s+([^ ]+)/) {
|
||||
my $script = uc($3);
|
||||
unless (exists $scriptCode{$script}) {
|
||||
warn "unknown ICU script $script";
|
||||
$scriptCode{$script} = $scriptCode{"UNKNOWN"};
|
||||
}
|
||||
$script = $scriptCode{$script};
|
||||
my $start = hex "0x$1";
|
||||
my $end = (defined $2) ? hex "0x$2" : $start;
|
||||
for (my $i = $start; $i <= $end; ++$i) {
|
||||
$script[$i] = $script;
|
||||
}
|
||||
}
|
||||
}
|
||||
close FH;
|
||||
|
||||
# read BidiMirroring.txt
|
||||
my @offsets = ();
|
||||
push @offsets, 0;
|
||||
|
||||
open FH, "< $UNICODE/BidiMirroring.txt" or die "can't open UCD file BidiMirroring.txt\n";
|
||||
push @versionInfo, "";
|
||||
while (<FH>) {
|
||||
chomp;
|
||||
push @versionInfo, $_;
|
||||
last if /Date:/;
|
||||
}
|
||||
while (<FH>) {
|
||||
s/#.*//;
|
||||
if (m/([0-9A-F]{4,6});\s*([0-9A-F]{4,6})/) {
|
||||
my $mirrorOffset = hex("0x$2") - hex("0x$1");
|
||||
my $offsetIndex = first { $offsets[$_] eq $mirrorOffset } 0..$#offsets;
|
||||
if ($offsetIndex == undef) {
|
||||
die "too many offset codes\n" if scalar @offsets == 31;
|
||||
push @offsets, $mirrorOffset;
|
||||
$offsetIndex = $#offsets;
|
||||
}
|
||||
$mirror[hex "0x$1"] = $offsetIndex;
|
||||
}
|
||||
}
|
||||
close FH;
|
||||
|
||||
# read BidiBrackets.txt
|
||||
my %pairedBracketTypeCode = (
|
||||
'N' => 0,
|
||||
'O' => 1,
|
||||
'C' => 2
|
||||
);
|
||||
open FH, "< $UNICODE/BidiBrackets.txt" or die "can't open UCD file BidiBrackets.txt\n";
|
||||
push @versionInfo, "";
|
||||
while (<FH>) {
|
||||
chomp;
|
||||
push @versionInfo, $_;
|
||||
last if /Date:/;
|
||||
}
|
||||
while (<FH>) {
|
||||
s/#.*//;
|
||||
if (m/([0-9A-F]{4,6});\s*([0-9A-F]{4,6});\s*(.)/) {
|
||||
my $mirroredChar = $offsets[$mirror[hex "0x$1"]] + hex "0x$1";
|
||||
die "bidi bracket does not match mirrored char\n" unless $mirroredChar == hex "0x$2";
|
||||
my $pbt = uc($3);
|
||||
warn "unknown Bidi Bracket type" unless exists $pairedBracketTypeCode{$pbt};
|
||||
$pairedBracketType[hex "0x$1"] = $pairedBracketTypeCode{$pbt};
|
||||
}
|
||||
}
|
||||
close FH;
|
||||
|
||||
# read HangulSyllableType.txt
|
||||
my %hangulType = (
|
||||
'L' => 0x01,
|
||||
'V' => 0x02,
|
||||
'T' => 0x04,
|
||||
'LV' => 0x03,
|
||||
'LVT' => 0x07
|
||||
);
|
||||
open FH, "< $UNICODE/HangulSyllableType.txt" or die "can't open UCD file HangulSyllableType.txt\n";
|
||||
push @versionInfo, "";
|
||||
while (<FH>) {
|
||||
chomp;
|
||||
push @versionInfo, $_;
|
||||
last if /Date:/;
|
||||
}
|
||||
while (<FH>) {
|
||||
s/#.*//;
|
||||
if (m/([0-9A-F]{4,6})(?:\.\.([0-9A-F]{4,6}))*\s*;\s*([^ ]+)/) {
|
||||
my $hangul = uc($3);
|
||||
warn "unknown Hangul syllable type" unless exists $hangulType{$hangul};
|
||||
$hangul = $hangulType{$hangul};
|
||||
my $start = hex "0x$1";
|
||||
my $end = (defined $2) ? hex "0x$2" : $start;
|
||||
for (my $i = $start; $i <= $end; ++$i) {
|
||||
$hangul[$i] = $hangul;
|
||||
}
|
||||
}
|
||||
}
|
||||
close FH;
|
||||
|
||||
# read LineBreak.txt
|
||||
open FH, "< $UNICODE/LineBreak.txt" or die "can't open UCD file LineBreak.txt\n";
|
||||
push @versionInfo, "";
|
||||
while (<FH>) {
|
||||
chomp;
|
||||
push @versionInfo, $_;
|
||||
last if /Date:/;
|
||||
}
|
||||
while (<FH>) {
|
||||
s/#.*//;
|
||||
if (m/([0-9A-F]{4,6})(?:\.\.([0-9A-F]{4,6}))*\s*;\s*([^ ]+)/) {
|
||||
my $lb = uc($3);
|
||||
warn "unknown LineBreak class" unless exists $lineBreakCode{$lb};
|
||||
$lb = $lineBreakCode{$lb};
|
||||
my $start = hex "0x$1";
|
||||
my $end = (defined $2) ? hex "0x$2" : $start;
|
||||
for (my $i = $start; $i <= $end; ++$i) {
|
||||
$lineBreak[$i] = $lb;
|
||||
}
|
||||
}
|
||||
}
|
||||
close FH;
|
||||
|
||||
# read EastAsianWidth.txt
|
||||
open FH, "< $UNICODE/EastAsianWidth.txt" or die "can't open UCD file EastAsianWidth.txt\n";
|
||||
push @versionInfo, "";
|
||||
while (<FH>) {
|
||||
chomp;
|
||||
push @versionInfo, $_;
|
||||
last if /Date:/;
|
||||
}
|
||||
while (<FH>) {
|
||||
s/#.*//;
|
||||
if (m/([0-9A-F]{4,6})(?:\.\.([0-9A-F]{4,6}))*\s*;\s*([^ ]+)/) {
|
||||
my $start = hex "0x$1";
|
||||
my $end = (defined $2) ? hex "0x$2" : $start;
|
||||
my $eaw = $3;
|
||||
warn "unknown EastAsianWidth class" unless exists $eastAsianWidthCode{$eaw};
|
||||
my $isFWH = ($eaw =~ m/^[FWH]$/) ? 1 : 0;
|
||||
for (my $i = $start; $i <= $end; ++$i) {
|
||||
$eastAsianWidthFWH[$i] = $isFWH;
|
||||
}
|
||||
}
|
||||
}
|
||||
close FH;
|
||||
|
||||
# read DerivedCoreProperties.txt (for Default-Ignorables)
|
||||
open FH, "< $UNICODE/DerivedCoreProperties.txt" or die "can't open UCD file DerivedCoreProperties.txt\n";
|
||||
push @versionInfo, "";
|
||||
while (<FH>) {
|
||||
chomp;
|
||||
push @versionInfo, $_;
|
||||
last if /Date:/;
|
||||
}
|
||||
while (<FH>) {
|
||||
s/#.*//;
|
||||
if (m/([0-9A-F]{4,6})(?:\.\.([0-9A-F]{4,6}))*\s*;\s*Default_Ignorable_Code_Point/) {
|
||||
my $start = hex "0x$1";
|
||||
my $end = (defined $2) ? hex "0x$2" : $start;
|
||||
for (my $i = $start; $i <= $end; ++$i) {
|
||||
$defaultIgnorable[$i] = 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
close FH;
|
||||
|
||||
# read IdentifierStatus.txt
|
||||
open FH, "< $UNICODE/security/IdentifierStatus.txt" or die "can't open UCD file IdentifierStatus.txt\n";
|
||||
push @versionInfo, "";
|
||||
|
|
@ -735,50 +343,17 @@ $versionInfo
|
|||
|
||||
__END
|
||||
|
||||
print DATA_TABLES "#if !ENABLE_INTL_API\n";
|
||||
print DATA_TABLES "static const uint32_t sScriptCodeToTag[] = {\n";
|
||||
for (my $i = 0; $i < scalar @scriptCodeToTag; ++$i) {
|
||||
printf DATA_TABLES " HB_TAG('%c','%c','%c','%c')", unpack('cccc', $scriptCodeToTag[$i]);
|
||||
print DATA_TABLES $i < $#scriptCodeToTag ? ",\n" : "\n";
|
||||
}
|
||||
print DATA_TABLES "};\n";
|
||||
print DATA_TABLES "#endif\n\n";
|
||||
|
||||
our $totalData = 0;
|
||||
|
||||
print DATA_TABLES "#if !ENABLE_INTL_API\n";
|
||||
print DATA_TABLES "static const int16_t sMirrorOffsets[] = {\n";
|
||||
for (my $i = 0; $i < scalar @offsets; ++$i) {
|
||||
printf DATA_TABLES " $offsets[$i]";
|
||||
print DATA_TABLES $i < $#offsets ? ",\n" : "\n";
|
||||
}
|
||||
print DATA_TABLES "};\n";
|
||||
print DATA_TABLES "#endif\n\n";
|
||||
|
||||
print HEADER "#pragma pack(1)\n\n";
|
||||
|
||||
sub sprintCharProps1
|
||||
{
|
||||
my $usv = shift;
|
||||
return sprintf("{%d,%d,%d}, ", $mirror[$usv], $hangul[$usv], $combining[$usv]);
|
||||
}
|
||||
my $type = q/
|
||||
struct nsCharProps1 {
|
||||
unsigned char mMirrorOffsetIndex:5;
|
||||
unsigned char mHangulType:3;
|
||||
unsigned char mCombiningClass:8;
|
||||
};
|
||||
/;
|
||||
&genTables("#if !ENABLE_INTL_API", "#endif",
|
||||
"CharProp1", $type, "nsCharProps1", 11, 5, \&sprintCharProps1, 1, 2, 1);
|
||||
|
||||
sub sprintCharProps2_short
|
||||
{
|
||||
my $usv = shift;
|
||||
return sprintf("{%d,%d},",
|
||||
$verticalOrientation[$usv], $idtype[$usv]);
|
||||
}
|
||||
$type = q|
|
||||
my $type = q|
|
||||
struct nsCharProps2 {
|
||||
// Currently only 4 bits are defined here, so 4 more could be added without
|
||||
// affecting the storage requirements for this struct. Or we could pack two
|
||||
|
|
@ -787,41 +362,7 @@ struct nsCharProps2 {
|
|||
unsigned char mIdType:2;
|
||||
};
|
||||
|;
|
||||
&genTables("#if ENABLE_INTL_API", "#endif",
|
||||
"CharProp2", $type, "nsCharProps2", 9, 7, \&sprintCharProps2_short, 16, 1, 1);
|
||||
|
||||
sub sprintCharProps2_full
|
||||
{
|
||||
my $usv = shift;
|
||||
return sprintf("{%d,%d,%d,%d,%d,%d,%d,%d,%d,%d},",
|
||||
$script[$usv], $pairedBracketType[$usv],
|
||||
$eastAsianWidthFWH[$usv], $category[$usv],
|
||||
$idtype[$usv], $defaultIgnorable[$usv], $bidicategory[$usv],
|
||||
$verticalOrientation[$usv], $lineBreak[$usv],
|
||||
$numericvalue[$usv]);
|
||||
}
|
||||
$type = q|
|
||||
// This struct currently requires 5 bytes. We try to ensure that whole-byte
|
||||
// fields will not straddle byte boundaries, to optimize access to them.
|
||||
struct nsCharProps2 {
|
||||
unsigned char mScriptCode:8;
|
||||
// -- byte boundary --
|
||||
unsigned char mPairedBracketType:2;
|
||||
unsigned char mEastAsianWidthFWH:1;
|
||||
unsigned char mCategory:5;
|
||||
// -- byte boundary --
|
||||
unsigned char mIdType:2;
|
||||
unsigned char mDefaultIgnorable:1;
|
||||
unsigned char mBidiCategory:5;
|
||||
// -- byte boundary --
|
||||
unsigned char mVertOrient:2;
|
||||
unsigned char mLineBreak:6;
|
||||
// -- byte boundary --
|
||||
signed char mNumericValue; // only 5 bits are actually needed here
|
||||
};
|
||||
|;
|
||||
&genTables("#if !ENABLE_INTL_API", "#endif",
|
||||
"CharProp2", $type, "nsCharProps2", 12, 4, \&sprintCharProps2_full, 16, 5, 1);
|
||||
&genTables("CharProp2", $type, "nsCharProps2", 9, 7, \&sprintCharProps2_short, 16, 1, 1);
|
||||
|
||||
print HEADER "#pragma pack()\n\n";
|
||||
|
||||
|
|
@ -837,50 +378,32 @@ sub sprintHanVariants
|
|||
return sprintf("0x%02x,", $val);
|
||||
}
|
||||
## Han Variant data currently unused but may be needed in future, see bug 857481
|
||||
## &genTables("", "", "HanVariant", "", "uint8_t", 9, 7, \&sprintHanVariants, 2, 1, 4);
|
||||
## &genTables("HanVariant", "", "uint8_t", 9, 7, \&sprintHanVariants, 2, 1, 4);
|
||||
|
||||
sub sprintFullWidth
|
||||
{
|
||||
my $usv = shift;
|
||||
return sprintf("0x%04x,", $fullWidth[$usv]);
|
||||
}
|
||||
&genTables("", "", "FullWidth", "", "uint16_t", 10, 6, \&sprintFullWidth, 0, 2, 1);
|
||||
&genTables("FullWidth", "", "uint16_t", 10, 6, \&sprintFullWidth, 0, 2, 1);
|
||||
|
||||
sub sprintFullWidthInverse
|
||||
{
|
||||
my $usv = shift;
|
||||
return sprintf("0x%04x,", $fullWidthInverse[$usv]);
|
||||
}
|
||||
&genTables("", "", "FullWidthInverse", "", "uint16_t", 10, 6, \&sprintFullWidthInverse, 0, 2, 1);
|
||||
|
||||
sub sprintCasemap
|
||||
{
|
||||
my $usv = shift;
|
||||
return sprintf("0x%08x,", $casemap[$usv]);
|
||||
}
|
||||
&genTables("#if !ENABLE_INTL_API", "#endif",
|
||||
"CaseMap", "", "uint32_t", 11, 5, \&sprintCasemap, 1, 4, 1);
|
||||
&genTables("FullWidthInverse", "", "uint16_t", 10, 6, \&sprintFullWidthInverse, 0, 2, 1);
|
||||
|
||||
print STDERR "Total data = $totalData\n";
|
||||
|
||||
printf DATA_TABLES "const uint32_t kTitleToUpper = 0x%08x;\n", $kTitleToUpper;
|
||||
printf DATA_TABLES "const uint32_t kUpperToLower = 0x%08x;\n", $kUpperToLower;
|
||||
printf DATA_TABLES "const uint32_t kLowerToTitle = 0x%08x;\n", $kLowerToTitle;
|
||||
printf DATA_TABLES "const uint32_t kLowerToUpper = 0x%08x;\n", $kLowerToUpper;
|
||||
printf DATA_TABLES "const uint32_t kCaseMapCharMask = 0x%08x;\n\n", $kCaseMapCharMask;
|
||||
|
||||
sub genTables
|
||||
{
|
||||
my ($guardBegin, $guardEnd,
|
||||
$prefix, $typedef, $type, $indexBits, $charBits, $func, $maxPlane, $bytesPerEntry, $charsPerEntry) = @_;
|
||||
my ($prefix, $typedef, $type, $indexBits, $charBits, $func, $maxPlane, $bytesPerEntry, $charsPerEntry) = @_;
|
||||
|
||||
if ($typedef ne '') {
|
||||
print HEADER "$guardBegin\n";
|
||||
print HEADER "$typedef\n";
|
||||
print HEADER "$guardEnd\n\n";
|
||||
}
|
||||
|
||||
print DATA_TABLES "\n$guardBegin\n";
|
||||
print DATA_TABLES "#define k${prefix}MaxPlane $maxPlane\n";
|
||||
print DATA_TABLES "#define k${prefix}IndexBits $indexBits\n";
|
||||
print DATA_TABLES "#define k${prefix}CharBits $charBits\n";
|
||||
|
|
@ -949,7 +472,6 @@ sub genTables
|
|||
print DATA_TABLES $i < $#char ? "},\n" : "}\n";
|
||||
}
|
||||
print DATA_TABLES "};\n";
|
||||
print DATA_TABLES "$guardEnd\n";
|
||||
|
||||
my $dataSize = $pmCount * $indexLen * $pmBits/8 +
|
||||
$chCount * $pageLen * $bytesPerEntry +
|
||||
|
|
|
|||
Загрузка…
Ссылка в новой задаче