gecko-dev/intl/uconv/tools/gen-big5-data.py

254 строки
5.4 KiB
Python

#!/usr/bin/python
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
# Adapted from
# https://hg.mozilla.org/projects/htmlparser/file/0d906fb1ab90/generate-encoding-data.py
# indexes.json comes from
# https://encoding.spec.whatwg.org/indexes.json
# i.e.
# https://github.com/whatwg/encoding/blob/ce4e83d0df5b5efec0697fc76e66699737e033a3/indexes.json
import json
indexes = json.load(open("indexes.json", "r"))
def nullToZero(codePoint):
if not codePoint:
codePoint = 0
return codePoint
index = []
for codePoint in indexes["big5"]:
index.append(nullToZero(codePoint))
# There are four major gaps consisting of more than 4 consecutive invalid pointers
gaps = []
consecutive = 0
consecutiveStart = 0
offset = 0
for codePoint in index:
if codePoint == 0:
if consecutive == 0:
consecutiveStart = offset
consecutive +=1
else:
if consecutive > 4:
gaps.append((consecutiveStart, consecutiveStart + consecutive))
consecutive = 0
offset += 1
def invertRanges(ranges, cap):
inverted = []
invertStart = 0
for (start, end) in ranges:
if start != 0:
inverted.append((invertStart, start))
invertStart = end
inverted.append((invertStart, cap))
return inverted
cap = len(index)
ranges = invertRanges(gaps, cap)
# Now compute a compressed lookup table for astralness
gaps = []
consecutive = 0
consecutiveStart = 0
offset = 0
for codePoint in index:
if codePoint <= 0xFFFF:
if consecutive == 0:
consecutiveStart = offset
consecutive +=1
else:
if consecutive > 40:
gaps.append((consecutiveStart, consecutiveStart + consecutive))
consecutive = 0
offset += 1
astralRanges = invertRanges(gaps, cap)
classFile = open("../ucvtw/nsBIG5Data.cpp", "w")
classFile.write('''/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
/*
* THIS IS A GENERATED FILE. PLEASE DO NOT EDIT.
* Instead, please regenerate using intl/uconv/tools/gen-big5-data.py
*/
#include "nsBIG5Data.h"
static const char16_t kBig5LowBitsTable[] = {
''')
for (low, high) in ranges:
for i in xrange(low, high):
classFile.write(' 0x%04X,\n' % (index[i] & 0xFFFF))
classFile.write('''};
static const uint32_t kBig5AstralnessTable[] = {
''')
# An array of bool is inefficient per
# http://stackoverflow.com/questions/4049156/1-bit-per-bool-in-array-c
bits = []
for (low, high) in astralRanges:
for i in xrange(low, high):
bits.append(1 if index[i] > 0xFFFF else 0)
# pad length to multiple of 32
for i in xrange(32 - (len(bits) % 32)):
bits.append(0)
i = 0
while i < len(bits):
accu = 0
for j in xrange(32):
accu |= bits[i + j] << j
classFile.write(' 0x%08X,\n' % accu)
i += 32
classFile.write('''};
// static
char16_t
nsBIG5Data::LowBits(size_t aPointer)
{
''')
base = 0
for (low, high) in ranges:
classFile.write(''' if (aPointer < %d) {
return 0;
}
if (aPointer < %d) {
return kBig5LowBitsTable[%d + (aPointer - %d)];
}
''' % (low, high, base, low))
base += (high - low)
classFile.write(''' return 0;
}
// static
bool
nsBIG5Data::IsAstral(size_t aPointer)
{
''')
base = 0
for (low, high) in astralRanges:
if high - low == 1:
classFile.write(''' if (aPointer < %d) {
return false;
}
if (aPointer == %d) {
return true;
}
''' % (low, low))
else:
classFile.write(''' if (aPointer < %d) {
return false;
}
if (aPointer < %d) {
size_t index = %d + (aPointer - %d);
return kBig5AstralnessTable[index >> 5] & (1 << (index & 0x1F));
}
''' % (low, high, base, low))
base += (high - low)
classFile.write(''' return false;
}
//static
size_t
nsBIG5Data::FindPointer(char16_t aLowBits, bool aIsAstral)
{
if (!aIsAstral) {
switch (aLowBits) {
''')
hkscsBound = (0xA1 - 0x81) * 157
preferLast = [
0x2550,
0x255E,
0x2561,
0x256A,
0x5341,
0x5345,
]
for codePoint in preferLast:
# Python lists don't have .rindex() :-(
for i in xrange(len(index) - 1, -1, -1):
candidate = index[i]
if candidate == codePoint:
classFile.write(''' case 0x%04X:
return %d;
''' % (codePoint, i))
break
classFile.write(''' default:
break;
}
}''')
base = 0
start = 0
for (low, high) in ranges:
if low <= hkscsBound and hkscsBound < high:
# This is the first range we don't ignore and the
# range that contains the first non-HKSCS pointer.
# Avoid searching HKSCS.
start = base + hkscsBound - low
break
base += (high - low)
classFile.write('''
for (size_t i = %d; i < MOZ_ARRAY_LENGTH(kBig5LowBitsTable); ++i) {
if (kBig5LowBitsTable[i] == aLowBits) {
size_t pointer;
''' % start)
base = 0
prevLow = 0
prevHigh = 0
prevBase = 0
writing = False
for (low, high) in ranges:
if writing:
classFile.write('''if (i < %d) {
pointer = i + %d;
} else ''' % ((prevBase + prevHigh - prevLow), (prevLow - prevBase)))
prevLow = low
prevHigh = high
prevBase = base
if high > hkscsBound:
writing = True
base += (high - low)
classFile.write('''{
pointer = i + %d;
}''' % (prevLow - prevBase))
classFile.write('''
if (aIsAstral == IsAstral(pointer)) {
return pointer;
}
}
}
return 0;
}
''')
classFile.close()