зеркало из https://github.com/mozilla/gecko-dev.git
Bug 1590167 - Add Rust implementation of hyphenation (mapped_hyph) and hook up in place of libhyphen. r=heycam
Differential Revision: https://phabricator.services.mozilla.com/D49967 --HG-- extra : moz-landing-system : lando
This commit is contained in:
Jonathan Kew
Родитель
b5ca94720e
Коммит
c01eaab5f5
|
|
@ -7,6 +7,11 @@ branch = "master"
|
|||
git = "https://github.com/mozilla/neqo"
|
||||
replace-with = "vendored-sources"
|
||||
|
||||
[source."https://github.com/jfkthame/mapped_hyph.git"]
|
||||
branch = "master"
|
||||
git = "https://github.com/jfkthame/mapped_hyph.git"
|
||||
replace-with = "vendored-sources"
|
||||
|
||||
[source."https://github.com/hsivonen/packed_simd"]
|
||||
branch = "rust_1_32"
|
||||
git = "https://github.com/hsivonen/packed_simd"
|
||||
|
|
|
|||
|
|
@ -1239,6 +1239,7 @@ dependencies = [
|
|||
"kvstore 0.1.0",
|
||||
"lmdb-rkv-sys 0.9.5 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"mapped_hyph 0.2.0 (git+https://github.com/jfkthame/mapped_hyph.git)",
|
||||
"mdns_service 0.1.0",
|
||||
"mozurl 0.0.1",
|
||||
"mp4parse_capi 0.11.2",
|
||||
|
|
@ -1706,6 +1707,15 @@ dependencies = [
|
|||
"synstructure 0.10.1 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "mapped_hyph"
|
||||
version = "0.2.0"
|
||||
source = "git+https://github.com/jfkthame/mapped_hyph.git#25d2359196071816cad2702ca7a278f0083f8bcd"
|
||||
dependencies = [
|
||||
"arrayref 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"memmap 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "marionette"
|
||||
version = "0.1.0"
|
||||
|
|
@ -2723,7 +2733,7 @@ dependencies = [
|
|||
"byteorder 1.3.1 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"digest 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"murmurhash3 0.0.5 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"rand 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"rand 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -4104,6 +4114,7 @@ dependencies = [
|
|||
"checksum lzw 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)" = "7d947cbb889ed21c2a84be6ffbaebf5b4e0f4340638cba0444907e38b56be084"
|
||||
"checksum mach 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "b823e83b2affd8f40a9ee8c29dbc56404c1e34cd2710921f2801e2cf29527afa"
|
||||
"checksum malloc_size_of_derive 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "35adee9ed962cf7d07d62cb58bc45029f3227f5b5b86246caa8632f06c187bc3"
|
||||
"checksum mapped_hyph 0.2.0 (git+https://github.com/jfkthame/mapped_hyph.git)" = "<none>"
|
||||
"checksum matches 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "100aabe6b8ff4e4a7e32c1c13523379802df0772b82466207ac25b013f193376"
|
||||
"checksum memchr 2.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "2efc7bc57c883d4a4d6e3246905283d8dae951bb3bd32f49d6ef297f546e1c39"
|
||||
"checksum memmap 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "6585fd95e7bb50d6cc31e20d4cf9afb4e2ba16c5846fc76793f11218da9c475b"
|
||||
|
|
|
|||
|
|
@ -1,46 +0,0 @@
|
|||
/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
|
||||
/* This Source Code Form is subject to the terms of the Mozilla Public
|
||||
* License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
|
||||
|
||||
/*
|
||||
* To enable us to load hyphenation dictionaries from arbitrary resource URIs,
|
||||
* not just through file paths using stdio, we override the (few) stdio APIs
|
||||
* that hyphen.c uses and provide our own reimplementation that calls Gecko
|
||||
* i/o methods.
|
||||
*/
|
||||
|
||||
#include <stdio.h> /* ensure stdio.h is loaded before our macros */
|
||||
|
||||
#undef FILE
|
||||
#define FILE hnjFile
|
||||
|
||||
#define fopen(path, mode) hnjFopen(path, mode)
|
||||
#define fclose(file) hnjFclose(file)
|
||||
#define fgets(buf, count, file) hnjFgets(buf, count, file)
|
||||
#define feof(file) hnjFeof(file)
|
||||
#define fgetc(file) hnjFgetc(file)
|
||||
|
||||
typedef struct hnjFile_ hnjFile;
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
void* hnj_malloc(size_t size);
|
||||
void* hnj_realloc(void* ptr, size_t size);
|
||||
void hnj_free(void* ptr);
|
||||
|
||||
hnjFile* hnjFopen(const char* aURISpec, const char* aMode);
|
||||
|
||||
int hnjFclose(hnjFile* f);
|
||||
|
||||
char* hnjFgets(char* s, int n, hnjFile* f);
|
||||
|
||||
int hnjFeof(hnjFile* f);
|
||||
|
||||
int hnjFgetc(hnjFile* f);
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
|
@ -1,133 +0,0 @@
|
|||
/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
|
||||
/* This Source Code Form is subject to the terms of the Mozilla Public
|
||||
* License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
|
||||
|
||||
// This file provides substitutes for the basic stdio routines used by hyphen.c
|
||||
// to read its dictionary files. We #define the stdio names to these versions
|
||||
// in hnjalloc.h, so that we can use nsIURI and nsIInputStream to specify and
|
||||
// access the dictionary resources.
|
||||
|
||||
#include "hnjalloc.h"
|
||||
|
||||
#undef FILE // Undo #defines from hnjalloc.h before #including other headers
|
||||
#undef fopen
|
||||
#undef fclose
|
||||
#undef fgets
|
||||
#undef feof
|
||||
#undef fgetc
|
||||
|
||||
#include "nsNetUtil.h"
|
||||
#include "nsIInputStream.h"
|
||||
#include "nsIURI.h"
|
||||
#include "nsContentUtils.h"
|
||||
|
||||
#define BUFSIZE 1024
|
||||
|
||||
struct hnjFile_ {
|
||||
nsCOMPtr<nsIInputStream> mStream;
|
||||
char mBuffer[BUFSIZE];
|
||||
uint32_t mCurPos;
|
||||
uint32_t mLimit;
|
||||
bool mEOF;
|
||||
};
|
||||
|
||||
// replacement for fopen()
|
||||
// (not a full substitute: only supports read access)
|
||||
hnjFile* hnjFopen(const char* aURISpec, const char* aMode) {
|
||||
// this override only needs to support "r"
|
||||
NS_ASSERTION(!strcmp(aMode, "r"), "unsupported fopen() mode in hnjFopen");
|
||||
|
||||
nsCOMPtr<nsIURI> uri;
|
||||
nsresult rv = NS_NewURI(getter_AddRefs(uri), aURISpec);
|
||||
if (NS_FAILED(rv)) {
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
nsCOMPtr<nsIChannel> channel;
|
||||
rv = NS_NewChannel(getter_AddRefs(channel), uri,
|
||||
nsContentUtils::GetSystemPrincipal(),
|
||||
nsILoadInfo::SEC_ALLOW_CROSS_ORIGIN_DATA_IS_NULL,
|
||||
nsIContentPolicy::TYPE_OTHER);
|
||||
if (NS_FAILED(rv)) {
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
nsCOMPtr<nsIInputStream> instream;
|
||||
rv = channel->Open(getter_AddRefs(instream));
|
||||
if (NS_FAILED(rv)) {
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
hnjFile* f = new hnjFile;
|
||||
f->mStream = instream;
|
||||
f->mCurPos = 0;
|
||||
f->mLimit = 0;
|
||||
f->mEOF = false;
|
||||
|
||||
return f;
|
||||
}
|
||||
|
||||
// replacement for fclose()
|
||||
int hnjFclose(hnjFile* f) {
|
||||
NS_ASSERTION(f && f->mStream, "bad argument to hnjFclose");
|
||||
|
||||
int result = 0;
|
||||
nsresult rv = f->mStream->Close();
|
||||
if (NS_FAILED(rv)) {
|
||||
result = EOF;
|
||||
}
|
||||
f->mStream = nullptr;
|
||||
|
||||
delete f;
|
||||
return result;
|
||||
}
|
||||
|
||||
// replacement for fgetc()
|
||||
int hnjFgetc(hnjFile* f) {
|
||||
if (f->mCurPos >= f->mLimit) {
|
||||
f->mCurPos = 0;
|
||||
|
||||
nsresult rv = f->mStream->Read(f->mBuffer, BUFSIZE, &f->mLimit);
|
||||
if (NS_FAILED(rv)) {
|
||||
f->mLimit = 0;
|
||||
}
|
||||
|
||||
if (f->mLimit == 0) {
|
||||
f->mEOF = true;
|
||||
return EOF;
|
||||
}
|
||||
}
|
||||
|
||||
return f->mBuffer[f->mCurPos++];
|
||||
}
|
||||
|
||||
// replacement for fgets()
|
||||
// (not a full reimplementation, but sufficient for libhyphen's needs)
|
||||
char* hnjFgets(char* s, int n, hnjFile* f) {
|
||||
NS_ASSERTION(s && f, "bad argument to hnjFgets");
|
||||
|
||||
int i = 0;
|
||||
while (i < n - 1) {
|
||||
int c = hnjFgetc(f);
|
||||
|
||||
if (c == EOF) {
|
||||
break;
|
||||
}
|
||||
|
||||
s[i++] = c;
|
||||
|
||||
if (c == '\n' || c == '\r') {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (i == 0) {
|
||||
return nullptr; // end of file
|
||||
}
|
||||
|
||||
s[i] = '\0'; // null-terminate the returned string
|
||||
return s;
|
||||
}
|
||||
|
||||
int hnjFeof(hnjFile* f) { return f->mEOF ? EOF : 0; }
|
||||
|
|
@ -14,16 +14,17 @@ UNIFIED_SOURCES += [
|
|||
'nsHyphenator.cpp',
|
||||
]
|
||||
|
||||
# These files cannot be built in unified mode because they include hnjalloc.h.
|
||||
SOURCES += [
|
||||
'hnjstdio.cpp',
|
||||
]
|
||||
|
||||
LOCAL_INCLUDES += [
|
||||
'../hyphen',
|
||||
]
|
||||
|
||||
FINAL_LIBRARY = 'xul'
|
||||
|
||||
if CONFIG['CC_TYPE'] in ('clang', 'gcc'):
|
||||
CXXFLAGS += ['-Wno-error=shadow']
|
||||
|
||||
GENERATED_FILES += [
|
||||
'mapped_hyph.h'
|
||||
]
|
||||
|
||||
generated = GENERATED_FILES['mapped_hyph.h']
|
||||
generated.script = '/layout/style/RunCbindgen.py:generate'
|
||||
generated.inputs = [
|
||||
'/third_party/rust/mapped_hyph'
|
||||
]
|
||||
|
|
|
|||
|
|
@ -37,8 +37,7 @@ static const char kMemoryPressureNotification[] = "memory-pressure";
|
|||
static const char kParentShuttingDownNotification[] = "profile-before-change";
|
||||
static const char kChildShuttingDownNotification[] = "content-child-shutdown";
|
||||
|
||||
class HyphenReporter final : public nsIMemoryReporter,
|
||||
public CountingAllocatorBase<HyphenReporter> {
|
||||
class HyphenReporter final : public nsIMemoryReporter {
|
||||
private:
|
||||
~HyphenReporter() = default;
|
||||
|
||||
|
|
@ -47,14 +46,19 @@ class HyphenReporter final : public nsIMemoryReporter,
|
|||
|
||||
// For telemetry, we report the memory rounded up to the nearest KB.
|
||||
static uint32_t MemoryAllocatedInKB() {
|
||||
return (MemoryAllocated() + 1023) / 1024;
|
||||
size_t total = 0;
|
||||
if (nsHyphenationManager::Instance()) {
|
||||
total = nsHyphenationManager::Instance()->SizeOfIncludingThis(
|
||||
moz_malloc_size_of);
|
||||
}
|
||||
return (total + 1023) / 1024;
|
||||
}
|
||||
|
||||
NS_IMETHOD CollectReports(nsIHandleReportCallback* aHandleReport,
|
||||
nsISupports* aData, bool aAnonymize) override {
|
||||
size_t total = MemoryAllocated();
|
||||
size_t total = 0;
|
||||
if (nsHyphenationManager::Instance()) {
|
||||
total += nsHyphenationManager::Instance()->SizeOfIncludingThis(
|
||||
total = nsHyphenationManager::Instance()->SizeOfIncludingThis(
|
||||
moz_malloc_size_of);
|
||||
}
|
||||
MOZ_COLLECT_REPORT("explicit/hyphenation", KIND_HEAP, UNITS_BYTES, total,
|
||||
|
|
@ -65,30 +69,6 @@ class HyphenReporter final : public nsIMemoryReporter,
|
|||
|
||||
NS_IMPL_ISUPPORTS(HyphenReporter, nsIMemoryReporter)
|
||||
|
||||
template <>
|
||||
CountingAllocatorBase<HyphenReporter>::AmountType
|
||||
CountingAllocatorBase<HyphenReporter>::sAmount(0);
|
||||
|
||||
/**
|
||||
* Allocation wrappers to track the amount of memory allocated by libhyphen.
|
||||
* Note that libhyphen assumes its malloc/realloc functions are infallible!
|
||||
*/
|
||||
extern "C" {
|
||||
void* hnj_malloc(size_t aSize);
|
||||
void* hnj_realloc(void* aPtr, size_t aSize);
|
||||
void hnj_free(void* aPtr);
|
||||
};
|
||||
|
||||
void* hnj_malloc(size_t aSize) {
|
||||
return HyphenReporter::InfallibleCountingMalloc(aSize);
|
||||
}
|
||||
|
||||
void* hnj_realloc(void* aPtr, size_t aSize) {
|
||||
return HyphenReporter::InfallibleCountingRealloc(aPtr, aSize);
|
||||
}
|
||||
|
||||
void hnj_free(void* aPtr) { HyphenReporter::CountingFree(aPtr); }
|
||||
|
||||
nsHyphenationManager* nsHyphenationManager::sInstance = nullptr;
|
||||
|
||||
NS_IMPL_ISUPPORTS(nsHyphenationManager, nsIObserver)
|
||||
|
|
@ -257,7 +237,7 @@ void nsHyphenationManager::LoadPatternListFromOmnijar(Omnijar::Type aType) {
|
|||
}
|
||||
|
||||
nsZipFind* find;
|
||||
zip->FindInit("hyphenation/hyph_*.dic", &find);
|
||||
zip->FindInit("hyphenation/hyph_*.hyf", &find);
|
||||
if (!find) {
|
||||
return;
|
||||
}
|
||||
|
|
@ -278,7 +258,7 @@ void nsHyphenationManager::LoadPatternListFromOmnijar(Omnijar::Type aType) {
|
|||
continue;
|
||||
}
|
||||
ToLowerCase(locale);
|
||||
locale.SetLength(locale.Length() - 4); // strip ".dic"
|
||||
locale.SetLength(locale.Length() - 4); // strip ".hyf"
|
||||
locale.Cut(0, locale.RFindChar('/') + 1); // strip directory
|
||||
if (StringBeginsWith(locale, NS_LITERAL_CSTRING("hyph_"))) {
|
||||
locale.Cut(0, 5);
|
||||
|
|
@ -323,13 +303,13 @@ void nsHyphenationManager::LoadPatternListFromDir(nsIFile* aDir) {
|
|||
file->GetLeafName(dictName);
|
||||
NS_ConvertUTF16toUTF8 locale(dictName);
|
||||
ToLowerCase(locale);
|
||||
if (!StringEndsWith(locale, NS_LITERAL_CSTRING(".dic"))) {
|
||||
if (!StringEndsWith(locale, NS_LITERAL_CSTRING(".hyf"))) {
|
||||
continue;
|
||||
}
|
||||
if (StringBeginsWith(locale, NS_LITERAL_CSTRING("hyph_"))) {
|
||||
locale.Cut(0, 5);
|
||||
}
|
||||
locale.SetLength(locale.Length() - 4); // strip ".dic"
|
||||
locale.SetLength(locale.Length() - 4); // strip ".hyf"
|
||||
for (uint32_t i = 0; i < locale.Length(); ++i) {
|
||||
if (locale[i] == '_') {
|
||||
locale.Replace(i, 1, '-');
|
||||
|
|
@ -383,9 +363,6 @@ size_t nsHyphenationManager::SizeOfIncludingThis(MallocSizeOf aMallocSizeOf) {
|
|||
// finds it is worthwhile.
|
||||
|
||||
result += mHyphenators.ShallowSizeOfExcludingThis(aMallocSizeOf);
|
||||
for (auto i = mHyphenators.ConstIter(); !i.Done(); i.Next()) {
|
||||
result += aMallocSizeOf(i.Data().get());
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -4,34 +4,94 @@
|
|||
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
|
||||
|
||||
#include "nsHyphenator.h"
|
||||
#include "nsIFile.h"
|
||||
#include "nsUTF8Utils.h"
|
||||
#include "nsUnicodeProperties.h"
|
||||
#include "nsIURI.h"
|
||||
#include "mozilla/Telemetry.h"
|
||||
|
||||
#include "hyphen.h"
|
||||
#include "mozilla/Telemetry.h"
|
||||
#include "nsIFile.h"
|
||||
#include "nsIFileURL.h"
|
||||
#include "nsIJARURI.h"
|
||||
#include "nsIURI.h"
|
||||
#include "nsNetUtil.h"
|
||||
#include "nsUnicodeProperties.h"
|
||||
#include "nsUTF8Utils.h"
|
||||
|
||||
#include "mapped_hyph.h"
|
||||
|
||||
static const void* GetItemPtrFromJarURI(nsIJARURI* aJAR, uint32_t* aLength) {
|
||||
// Try to get the jarfile's nsZipArchive, find the relevant item, and return
|
||||
// a pointer to its data provided it is stored uncompressed.
|
||||
nsCOMPtr<nsIURI> jarFile;
|
||||
if (NS_FAILED(aJAR->GetJARFile(getter_AddRefs(jarFile)))) {
|
||||
return nullptr;
|
||||
}
|
||||
nsCOMPtr<nsIFileURL> fileUrl = do_QueryInterface(jarFile);
|
||||
if (!fileUrl) {
|
||||
return nullptr;
|
||||
}
|
||||
nsCOMPtr<nsIFile> file;
|
||||
fileUrl->GetFile(getter_AddRefs(file));
|
||||
if (!file) {
|
||||
return nullptr;
|
||||
}
|
||||
RefPtr<nsZipArchive> archive = mozilla::Omnijar::GetReader(file);
|
||||
if (archive) {
|
||||
nsCString path;
|
||||
aJAR->GetJAREntry(path);
|
||||
nsZipItem* item = archive->GetItem(path.get());
|
||||
if (item && item->Compression() == 0 && item->Size() > 0) {
|
||||
// We do NOT own this data, but it won't go away until the omnijar
|
||||
// file is closed during shutdown.
|
||||
const uint8_t* data = archive->GetData(item);
|
||||
if (data) {
|
||||
*aLength = item->Size();
|
||||
return data;
|
||||
}
|
||||
}
|
||||
}
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
nsHyphenator::nsHyphenator(nsIURI* aURI, bool aHyphenateCapitalized)
|
||||
: mDict(nullptr), mHyphenateCapitalized(aHyphenateCapitalized) {
|
||||
nsCString uriSpec;
|
||||
nsresult rv = aURI->GetSpec(uriSpec);
|
||||
if (NS_FAILED(rv)) {
|
||||
: mDict(nullptr),
|
||||
mDictSize(0),
|
||||
mHyphenateCapitalized(aHyphenateCapitalized) {
|
||||
Telemetry::AutoTimer<Telemetry::HYPHENATION_LOAD_TIME> telemetry;
|
||||
|
||||
nsCOMPtr<nsIJARURI> jar = do_QueryInterface(aURI);
|
||||
if (jar) {
|
||||
// This gives us a raw pointer into the omnijar's data; we do not own
|
||||
// it and must not attempt to free it!
|
||||
mDict = GetItemPtrFromJarURI(jar, &mDictSize);
|
||||
// Reject the resource if it fails to validate.
|
||||
if (!mDict || !mDictSize ||
|
||||
!mapped_hyph_is_valid_hyphenator(static_cast<const uint8_t*>(mDict),
|
||||
mDictSize)) {
|
||||
MOZ_ASSERT_UNREACHABLE("invalid packaged hyphenation resource?");
|
||||
mDict = nullptr;
|
||||
mDictSize = 0;
|
||||
}
|
||||
return;
|
||||
}
|
||||
Telemetry::AutoTimer<Telemetry::HYPHENATION_LOAD_TIME> telemetry;
|
||||
mDict = hnj_hyphen_load(uriSpec.get());
|
||||
#ifdef DEBUG
|
||||
if (mDict) {
|
||||
printf("loaded hyphenation patterns from %s\n", uriSpec.get());
|
||||
|
||||
if (mozilla::net::SchemeIsFile(aURI)) {
|
||||
// Ask the Rust lib to mmap the file. In this case our mDictSize field
|
||||
// remains zero; mDict is not a pointer to the raw data but an opaque
|
||||
// reference to a Rust object, and can only be freed by passing it to
|
||||
// mapped_hyph_free_dictionary().
|
||||
nsAutoCString path;
|
||||
aURI->GetFilePath(path);
|
||||
mDict = mapped_hyph_load_dictionary(path.get());
|
||||
MOZ_ASSERT(mDict, "failed to load hyphenation resource from file");
|
||||
return;
|
||||
}
|
||||
|
||||
if (!mDict) {
|
||||
MOZ_ASSERT_UNREACHABLE("invalid hyphenation resource?");
|
||||
}
|
||||
#endif
|
||||
}
|
||||
|
||||
nsHyphenator::~nsHyphenator() {
|
||||
if (mDict != nullptr) {
|
||||
hnj_hyphen_free((HyphenDict*)mDict);
|
||||
mDict = nullptr;
|
||||
if (mDict && mDictSize == 0) {
|
||||
mapped_hyph_free_dictionary((HyphDic*)mDict);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -83,13 +143,12 @@ nsresult nsHyphenator::Hyphenate(const nsAString& aString,
|
|||
|
||||
void nsHyphenator::HyphenateWord(const nsAString& aString, uint32_t aStart,
|
||||
uint32_t aLimit, nsTArray<bool>& aHyphens) {
|
||||
// Convert word from aStart and aLimit in aString to utf-8 for libhyphen,
|
||||
// Convert word from aStart and aLimit in aString to utf-8 for mapped_hyph,
|
||||
// lowercasing it as we go so that it will match the (lowercased) patterns
|
||||
// (bug 1105644).
|
||||
nsAutoCString utf8;
|
||||
const char16_t* const begin = aString.BeginReading();
|
||||
const char16_t* cur = begin + aStart;
|
||||
const char16_t* end = begin + aLimit;
|
||||
const char16_t* cur = aString.BeginReading() + aStart;
|
||||
const char16_t* end = aString.BeginReading() + aLimit;
|
||||
bool firstLetter = true;
|
||||
while (cur < end) {
|
||||
uint32_t ch = *cur++;
|
||||
|
|
@ -98,10 +157,10 @@ void nsHyphenator::HyphenateWord(const nsAString& aString, uint32_t aStart,
|
|||
if (cur < end && NS_IS_LOW_SURROGATE(*cur)) {
|
||||
ch = SURROGATE_TO_UCS4(ch, *cur++);
|
||||
} else {
|
||||
ch = 0xfffd; // unpaired surrogate, treat as REPLACEMENT CHAR
|
||||
return; // unpaired surrogate: bail out, don't hyphenate broken text
|
||||
}
|
||||
} else if (NS_IS_LOW_SURROGATE(ch)) {
|
||||
ch = 0xfffd; // unpaired surrogate
|
||||
return; // unpaired surrogate
|
||||
}
|
||||
|
||||
// XXX What about language-specific casing? Consider Turkish I/i...
|
||||
|
|
@ -111,15 +170,11 @@ void nsHyphenator::HyphenateWord(const nsAString& aString, uint32_t aStart,
|
|||
ch = ToLowerCase(ch);
|
||||
|
||||
if (ch != origCh) {
|
||||
if (firstLetter) {
|
||||
// Avoid hyphenating capitalized words (bug 1550532) unless explicitly
|
||||
// allowed by prefs for the language in use.
|
||||
if (!mHyphenateCapitalized) {
|
||||
return;
|
||||
}
|
||||
} else {
|
||||
// Also never auto-hyphenate a word that has internal caps, as it may
|
||||
// well be an all-caps acronym or a quirky name like iTunes.
|
||||
// Avoid hyphenating capitalized words (bug 1550532) unless explicitly
|
||||
// allowed by prefs for the language in use.
|
||||
// Also never auto-hyphenate a word that has internal caps, as it may
|
||||
// well be an all-caps acronym or a quirky name like iTunes.
|
||||
if (!mHyphenateCapitalized || !firstLetter) {
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
|
@ -142,31 +197,43 @@ void nsHyphenator::HyphenateWord(const nsAString& aString, uint32_t aStart,
|
|||
}
|
||||
}
|
||||
|
||||
AutoTArray<char, 200> utf8hyphens;
|
||||
utf8hyphens.SetLength(utf8.Length() + 5);
|
||||
char** rep = nullptr;
|
||||
int* pos = nullptr;
|
||||
int* cut = nullptr;
|
||||
int err = hnj_hyphen_hyphenate2((HyphenDict*)mDict, utf8.BeginReading(),
|
||||
utf8.Length(), utf8hyphens.Elements(),
|
||||
nullptr, &rep, &pos, &cut);
|
||||
if (!err) {
|
||||
// Surprisingly, hnj_hyphen_hyphenate2 converts the 'hyphens' buffer
|
||||
// from utf8 code unit indexing (which would match the utf8 input
|
||||
// string directly) to Unicode character indexing.
|
||||
// We then need to convert this to utf16 code unit offsets for Gecko.
|
||||
const char* hyphPtr = utf8hyphens.Elements();
|
||||
const char16_t* cur = begin + aStart;
|
||||
const char16_t* end = begin + aLimit;
|
||||
while (cur < end) {
|
||||
if (*hyphPtr & 0x01) {
|
||||
aHyphens[cur - begin] = true;
|
||||
AutoTArray<uint8_t, 200> hyphenValues;
|
||||
hyphenValues.SetLength(utf8.Length());
|
||||
int32_t result;
|
||||
if (mDictSize > 0) {
|
||||
result = mapped_hyph_find_hyphen_values_raw(
|
||||
static_cast<const uint8_t*>(mDict), mDictSize, utf8.BeginReading(),
|
||||
utf8.Length(), hyphenValues.Elements(), hyphenValues.Length());
|
||||
} else {
|
||||
result = mapped_hyph_find_hyphen_values_dic(
|
||||
static_cast<const HyphDic*>(mDict), utf8.BeginReading(), utf8.Length(),
|
||||
hyphenValues.Elements(), hyphenValues.Length());
|
||||
}
|
||||
if (result > 0) {
|
||||
// We need to convert UTF-8 indexing as used by the hyphenation lib into
|
||||
// UTF-16 indexing of the aHyphens[] array for Gecko.
|
||||
uint32_t utf16index = 0;
|
||||
for (uint32_t utf8index = 0; utf8index < utf8.Length();) {
|
||||
// We know utf8 is valid, so we only need to look at the first byte of
|
||||
// each character to determine its length and the corresponding UTF-16
|
||||
// length to add to utf16index.
|
||||
const uint8_t leadByte = utf8[utf8index];
|
||||
if (leadByte < 0x80) {
|
||||
utf8index += 1;
|
||||
} else if (leadByte < 0xE0) {
|
||||
utf8index += 2;
|
||||
} else if (leadByte < 0xF0) {
|
||||
utf8index += 3;
|
||||
} else {
|
||||
utf8index += 4;
|
||||
}
|
||||
cur++;
|
||||
if (cur < end && NS_IS_SURROGATE_PAIR(*(cur - 1), *cur)) {
|
||||
cur++;
|
||||
// The hyphenation value of interest is the one for the last code unit
|
||||
// of the utf-8 character, and is recorded on the last code unit of the
|
||||
// utf-16 character (in the case of a surrogate pair).
|
||||
utf16index += leadByte >= 0xF0 ? 2 : 1;
|
||||
if (utf16index > 0 && (hyphenValues[utf8index - 1] & 0x01)) {
|
||||
aHyphens[aStart + utf16index - 1] = true;
|
||||
}
|
||||
hyphPtr++;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -28,7 +28,13 @@ class nsHyphenator {
|
|||
void HyphenateWord(const nsAString& aString, uint32_t aStart, uint32_t aLimit,
|
||||
nsTArray<bool>& aHyphens);
|
||||
|
||||
void* mDict;
|
||||
const void* mDict; // If mDictSize > 0, this points to a raw byte buffer
|
||||
// containing the hyphenation dictionary data (in the
|
||||
// memory-mapped omnijar);
|
||||
// if mDictSize == 0, it's a HyphDic reference created
|
||||
// by mapped_hyph_load_dictionary() and must be released
|
||||
// by calling mapped_hyph_free_dictionary().
|
||||
uint32_t mDictSize;
|
||||
bool mHyphenateCapitalized;
|
||||
};
|
||||
|
||||
|
|
|
|||
|
|
@ -1,17 +0,0 @@
|
|||
Libhnj was written by Raph Levien <raph at acm dot org>.
|
||||
|
||||
Original Libhnj source with OOo's patches are managed by Rene Engelhard and
|
||||
Chris Halls at Debian: http://packages.debian.org/stable/libdevel/libhnj-dev
|
||||
and http://packages.debian.org/unstable/source/libhnj
|
||||
|
||||
This subset of Libhnj was extended by
|
||||
Peter Novodvorsky <nidd at alt-linux dot org> (OOo integration),
|
||||
László Németh <nemeth at numbertext dot org> (non-standard and compound
|
||||
hyphenation with Unicode support),
|
||||
Nanning Buitenhuis <nanning at elvenkind dot com> (substrings.c)
|
||||
|
||||
Write bug reports to László Németh or in the bug tracker of hunspell.sf.net.
|
||||
|
||||
---
|
||||
Please contact Raph Levien for information about licensing for
|
||||
proprietary applications.
|
||||
|
|
@ -1,17 +0,0 @@
|
|||
GPL 2.0/LGPL 2.1/MPL 1.1 tri-license
|
||||
|
||||
The contents of this software may be used under the terms of
|
||||
the GNU General Public License Version 2 or later (the "GPL"), or
|
||||
the GNU Lesser General Public License Version 2.1 or later (the "LGPL",
|
||||
see COPYING.LGPL) or the Mozilla Public License Version 1.1 or later
|
||||
(the "MPL", see COPYING.MPL).
|
||||
|
||||
The Plain TeX hyphenation tables "hyphen.tex" by Donald E. Knuth
|
||||
has a non MPL/LGPL compatible license, but freely redistributable:
|
||||
"Unlimited copying and redistribution of this file are permitted as long
|
||||
as this file is not modified. Modifications are permitted, but only if
|
||||
the resulting file is not named hyphen.tex."
|
||||
|
||||
Software distributed under these licenses is distributed on an "AS IS" basis,
|
||||
WITHOUT WARRANTY OF ANY KIND, either express or implied. See the licences
|
||||
for the specific language governing rights and limitations under the licenses.
|
||||
|
|
@ -1,515 +0,0 @@
|
|||
|
||||
GNU LESSER GENERAL PUBLIC LICENSE
|
||||
Version 2.1, February 1999
|
||||
|
||||
Copyright (C) 1991, 1999 Free Software Foundation, Inc.
|
||||
59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
||||
Everyone is permitted to copy and distribute verbatim copies
|
||||
of this license document, but changing it is not allowed.
|
||||
|
||||
[This is the first released version of the Lesser GPL. It also counts
|
||||
as the successor of the GNU Library Public License, version 2, hence
|
||||
the version number 2.1.]
|
||||
|
||||
Preamble
|
||||
|
||||
The licenses for most software are designed to take away your
|
||||
freedom to share and change it. By contrast, the GNU General Public
|
||||
Licenses are intended to guarantee your freedom to share and change
|
||||
free software--to make sure the software is free for all its users.
|
||||
|
||||
This license, the Lesser General Public License, applies to some
|
||||
specially designated software packages--typically libraries--of the
|
||||
Free Software Foundation and other authors who decide to use it. You
|
||||
can use it too, but we suggest you first think carefully about whether
|
||||
this license or the ordinary General Public License is the better
|
||||
strategy to use in any particular case, based on the explanations
|
||||
below.
|
||||
|
||||
When we speak of free software, we are referring to freedom of use,
|
||||
not price. Our General Public Licenses are designed to make sure that
|
||||
you have the freedom to distribute copies of free software (and charge
|
||||
for this service if you wish); that you receive source code or can get
|
||||
it if you want it; that you can change the software and use pieces of
|
||||
it in new free programs; and that you are informed that you can do
|
||||
these things.
|
||||
|
||||
To protect your rights, we need to make restrictions that forbid
|
||||
distributors to deny you these rights or to ask you to surrender these
|
||||
rights. These restrictions translate to certain responsibilities for
|
||||
you if you distribute copies of the library or if you modify it.
|
||||
|
||||
For example, if you distribute copies of the library, whether gratis
|
||||
or for a fee, you must give the recipients all the rights that we gave
|
||||
you. You must make sure that they, too, receive or can get the source
|
||||
code. If you link other code with the library, you must provide
|
||||
complete object files to the recipients, so that they can relink them
|
||||
with the library after making changes to the library and recompiling
|
||||
it. And you must show them these terms so they know their rights.
|
||||
|
||||
We protect your rights with a two-step method: (1) we copyright the
|
||||
library, and (2) we offer you this license, which gives you legal
|
||||
permission to copy, distribute and/or modify the library.
|
||||
|
||||
To protect each distributor, we want to make it very clear that
|
||||
there is no warranty for the free library. Also, if the library is
|
||||
modified by someone else and passed on, the recipients should know
|
||||
that what they have is not the original version, so that the original
|
||||
author's reputation will not be affected by problems that might be
|
||||
introduced by others.
|
||||
^L
|
||||
Finally, software patents pose a constant threat to the existence of
|
||||
any free program. We wish to make sure that a company cannot
|
||||
effectively restrict the users of a free program by obtaining a
|
||||
restrictive license from a patent holder. Therefore, we insist that
|
||||
any patent license obtained for a version of the library must be
|
||||
consistent with the full freedom of use specified in this license.
|
||||
|
||||
Most GNU software, including some libraries, is covered by the
|
||||
ordinary GNU General Public License. This license, the GNU Lesser
|
||||
General Public License, applies to certain designated libraries, and
|
||||
is quite different from the ordinary General Public License. We use
|
||||
this license for certain libraries in order to permit linking those
|
||||
libraries into non-free programs.
|
||||
|
||||
When a program is linked with a library, whether statically or using
|
||||
a shared library, the combination of the two is legally speaking a
|
||||
combined work, a derivative of the original library. The ordinary
|
||||
General Public License therefore permits such linking only if the
|
||||
entire combination fits its criteria of freedom. The Lesser General
|
||||
Public License permits more lax criteria for linking other code with
|
||||
the library.
|
||||
|
||||
We call this license the "Lesser" General Public License because it
|
||||
does Less to protect the user's freedom than the ordinary General
|
||||
Public License. It also provides other free software developers Less
|
||||
of an advantage over competing non-free programs. These disadvantages
|
||||
are the reason we use the ordinary General Public License for many
|
||||
libraries. However, the Lesser license provides advantages in certain
|
||||
special circumstances.
|
||||
|
||||
For example, on rare occasions, there may be a special need to
|
||||
encourage the widest possible use of a certain library, so that it
|
||||
becomes
|
||||
a de-facto standard. To achieve this, non-free programs must be
|
||||
allowed to use the library. A more frequent case is that a free
|
||||
library does the same job as widely used non-free libraries. In this
|
||||
case, there is little to gain by limiting the free library to free
|
||||
software only, so we use the Lesser General Public License.
|
||||
|
||||
In other cases, permission to use a particular library in non-free
|
||||
programs enables a greater number of people to use a large body of
|
||||
free software. For example, permission to use the GNU C Library in
|
||||
non-free programs enables many more people to use the whole GNU
|
||||
operating system, as well as its variant, the GNU/Linux operating
|
||||
system.
|
||||
|
||||
Although the Lesser General Public License is Less protective of the
|
||||
users' freedom, it does ensure that the user of a program that is
|
||||
linked with the Library has the freedom and the wherewithal to run
|
||||
that program using a modified version of the Library.
|
||||
|
||||
The precise terms and conditions for copying, distribution and
|
||||
modification follow. Pay close attention to the difference between a
|
||||
"work based on the library" and a "work that uses the library". The
|
||||
former contains code derived from the library, whereas the latter must
|
||||
be combined with the library in order to run.
|
||||
^L
|
||||
GNU LESSER GENERAL PUBLIC LICENSE
|
||||
TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
|
||||
|
||||
0. This License Agreement applies to any software library or other
|
||||
program which contains a notice placed by the copyright holder or
|
||||
other authorized party saying it may be distributed under the terms of
|
||||
this Lesser General Public License (also called "this License").
|
||||
Each licensee is addressed as "you".
|
||||
|
||||
A "library" means a collection of software functions and/or data
|
||||
prepared so as to be conveniently linked with application programs
|
||||
(which use some of those functions and data) to form executables.
|
||||
|
||||
The "Library", below, refers to any such software library or work
|
||||
which has been distributed under these terms. A "work based on the
|
||||
Library" means either the Library or any derivative work under
|
||||
copyright law: that is to say, a work containing the Library or a
|
||||
portion of it, either verbatim or with modifications and/or translated
|
||||
straightforwardly into another language. (Hereinafter, translation is
|
||||
included without limitation in the term "modification".)
|
||||
|
||||
"Source code" for a work means the preferred form of the work for
|
||||
making modifications to it. For a library, complete source code means
|
||||
all the source code for all modules it contains, plus any associated
|
||||
interface definition files, plus the scripts used to control
|
||||
compilation
|
||||
and installation of the library.
|
||||
|
||||
Activities other than copying, distribution and modification are not
|
||||
covered by this License; they are outside its scope. The act of
|
||||
running a program using the Library is not restricted, and output from
|
||||
such a program is covered only if its contents constitute a work based
|
||||
on the Library (independent of the use of the Library in a tool for
|
||||
writing it). Whether that is true depends on what the Library does
|
||||
and what the program that uses the Library does.
|
||||
|
||||
1. You may copy and distribute verbatim copies of the Library's
|
||||
complete source code as you receive it, in any medium, provided that
|
||||
you conspicuously and appropriately publish on each copy an
|
||||
appropriate copyright notice and disclaimer of warranty; keep intact
|
||||
all the notices that refer to this License and to the absence of any
|
||||
warranty; and distribute a copy of this License along with the
|
||||
Library.
|
||||
|
||||
You may charge a fee for the physical act of transferring a copy,
|
||||
and you may at your option offer warranty protection in exchange for a
|
||||
fee.
|
||||
|
||||
2. You may modify your copy or copies of the Library or any portion
|
||||
of it, thus forming a work based on the Library, and copy and
|
||||
distribute such modifications or work under the terms of Section 1
|
||||
above, provided that you also meet all of these conditions:
|
||||
|
||||
a) The modified work must itself be a software library.
|
||||
|
||||
b) You must cause the files modified to carry prominent notices
|
||||
stating that you changed the files and the date of any change.
|
||||
|
||||
c) You must cause the whole of the work to be licensed at no
|
||||
charge to all third parties under the terms of this License.
|
||||
|
||||
d) If a facility in the modified Library refers to a function or a
|
||||
table of data to be supplied by an application program that uses
|
||||
the facility, other than as an argument passed when the facility
|
||||
is invoked, then you must make a good faith effort to ensure that,
|
||||
in the event an application does not supply such function or
|
||||
table, the facility still operates, and performs whatever part of
|
||||
its purpose remains meaningful.
|
||||
|
||||
(For example, a function in a library to compute square roots has
|
||||
a purpose that is entirely well-defined independent of the
|
||||
application. Therefore, Subsection 2d requires that any
|
||||
application-supplied function or table used by this function must
|
||||
be optional: if the application does not supply it, the square
|
||||
root function must still compute square roots.)
|
||||
|
||||
These requirements apply to the modified work as a whole. If
|
||||
identifiable sections of that work are not derived from the Library,
|
||||
and can be reasonably considered independent and separate works in
|
||||
themselves, then this License, and its terms, do not apply to those
|
||||
sections when you distribute them as separate works. But when you
|
||||
distribute the same sections as part of a whole which is a work based
|
||||
on the Library, the distribution of the whole must be on the terms of
|
||||
this License, whose permissions for other licensees extend to the
|
||||
entire whole, and thus to each and every part regardless of who wrote
|
||||
it.
|
||||
|
||||
Thus, it is not the intent of this section to claim rights or contest
|
||||
your rights to work written entirely by you; rather, the intent is to
|
||||
exercise the right to control the distribution of derivative or
|
||||
collective works based on the Library.
|
||||
|
||||
In addition, mere aggregation of another work not based on the Library
|
||||
with the Library (or with a work based on the Library) on a volume of
|
||||
a storage or distribution medium does not bring the other work under
|
||||
the scope of this License.
|
||||
|
||||
3. You may opt to apply the terms of the ordinary GNU General Public
|
||||
License instead of this License to a given copy of the Library. To do
|
||||
this, you must alter all the notices that refer to this License, so
|
||||
that they refer to the ordinary GNU General Public License, version 2,
|
||||
instead of to this License. (If a newer version than version 2 of the
|
||||
ordinary GNU General Public License has appeared, then you can specify
|
||||
that version instead if you wish.) Do not make any other change in
|
||||
these notices.
|
||||
^L
|
||||
Once this change is made in a given copy, it is irreversible for
|
||||
that copy, so the ordinary GNU General Public License applies to all
|
||||
subsequent copies and derivative works made from that copy.
|
||||
|
||||
This option is useful when you wish to copy part of the code of
|
||||
the Library into a program that is not a library.
|
||||
|
||||
4. You may copy and distribute the Library (or a portion or
|
||||
derivative of it, under Section 2) in object code or executable form
|
||||
under the terms of Sections 1 and 2 above provided that you accompany
|
||||
it with the complete corresponding machine-readable source code, which
|
||||
must be distributed under the terms of Sections 1 and 2 above on a
|
||||
medium customarily used for software interchange.
|
||||
|
||||
If distribution of object code is made by offering access to copy
|
||||
from a designated place, then offering equivalent access to copy the
|
||||
source code from the same place satisfies the requirement to
|
||||
distribute the source code, even though third parties are not
|
||||
compelled to copy the source along with the object code.
|
||||
|
||||
5. A program that contains no derivative of any portion of the
|
||||
Library, but is designed to work with the Library by being compiled or
|
||||
linked with it, is called a "work that uses the Library". Such a
|
||||
work, in isolation, is not a derivative work of the Library, and
|
||||
therefore falls outside the scope of this License.
|
||||
|
||||
However, linking a "work that uses the Library" with the Library
|
||||
creates an executable that is a derivative of the Library (because it
|
||||
contains portions of the Library), rather than a "work that uses the
|
||||
library". The executable is therefore covered by this License.
|
||||
Section 6 states terms for distribution of such executables.
|
||||
|
||||
When a "work that uses the Library" uses material from a header file
|
||||
that is part of the Library, the object code for the work may be a
|
||||
derivative work of the Library even though the source code is not.
|
||||
Whether this is true is especially significant if the work can be
|
||||
linked without the Library, or if the work is itself a library. The
|
||||
threshold for this to be true is not precisely defined by law.
|
||||
|
||||
If such an object file uses only numerical parameters, data
|
||||
structure layouts and accessors, and small macros and small inline
|
||||
functions (ten lines or less in length), then the use of the object
|
||||
file is unrestricted, regardless of whether it is legally a derivative
|
||||
work. (Executables containing this object code plus portions of the
|
||||
Library will still fall under Section 6.)
|
||||
|
||||
Otherwise, if the work is a derivative of the Library, you may
|
||||
distribute the object code for the work under the terms of Section 6.
|
||||
Any executables containing that work also fall under Section 6,
|
||||
whether or not they are linked directly with the Library itself.
|
||||
^L
|
||||
6. As an exception to the Sections above, you may also combine or
|
||||
link a "work that uses the Library" with the Library to produce a
|
||||
work containing portions of the Library, and distribute that work
|
||||
under terms of your choice, provided that the terms permit
|
||||
modification of the work for the customer's own use and reverse
|
||||
engineering for debugging such modifications.
|
||||
|
||||
You must give prominent notice with each copy of the work that the
|
||||
Library is used in it and that the Library and its use are covered by
|
||||
this License. You must supply a copy of this License. If the work
|
||||
during execution displays copyright notices, you must include the
|
||||
copyright notice for the Library among them, as well as a reference
|
||||
directing the user to the copy of this License. Also, you must do one
|
||||
of these things:
|
||||
|
||||
a) Accompany the work with the complete corresponding
|
||||
machine-readable source code for the Library including whatever
|
||||
changes were used in the work (which must be distributed under
|
||||
Sections 1 and 2 above); and, if the work is an executable linked
|
||||
with the Library, with the complete machine-readable "work that
|
||||
uses the Library", as object code and/or source code, so that the
|
||||
user can modify the Library and then relink to produce a modified
|
||||
executable containing the modified Library. (It is understood
|
||||
that the user who changes the contents of definitions files in the
|
||||
Library will not necessarily be able to recompile the application
|
||||
to use the modified definitions.)
|
||||
|
||||
b) Use a suitable shared library mechanism for linking with the
|
||||
Library. A suitable mechanism is one that (1) uses at run time a
|
||||
copy of the library already present on the user's computer system,
|
||||
rather than copying library functions into the executable, and (2)
|
||||
will operate properly with a modified version of the library, if
|
||||
the user installs one, as long as the modified version is
|
||||
interface-compatible with the version that the work was made with.
|
||||
|
||||
c) Accompany the work with a written offer, valid for at
|
||||
least three years, to give the same user the materials
|
||||
specified in Subsection 6a, above, for a charge no more
|
||||
than the cost of performing this distribution.
|
||||
|
||||
d) If distribution of the work is made by offering access to copy
|
||||
from a designated place, offer equivalent access to copy the above
|
||||
specified materials from the same place.
|
||||
|
||||
e) Verify that the user has already received a copy of these
|
||||
materials or that you have already sent this user a copy.
|
||||
|
||||
For an executable, the required form of the "work that uses the
|
||||
Library" must include any data and utility programs needed for
|
||||
reproducing the executable from it. However, as a special exception,
|
||||
the materials to be distributed need not include anything that is
|
||||
normally distributed (in either source or binary form) with the major
|
||||
components (compiler, kernel, and so on) of the operating system on
|
||||
which the executable runs, unless that component itself accompanies
|
||||
the executable.
|
||||
|
||||
It may happen that this requirement contradicts the license
|
||||
restrictions of other proprietary libraries that do not normally
|
||||
accompany the operating system. Such a contradiction means you cannot
|
||||
use both them and the Library together in an executable that you
|
||||
distribute.
|
||||
^L
|
||||
7. You may place library facilities that are a work based on the
|
||||
Library side-by-side in a single library together with other library
|
||||
facilities not covered by this License, and distribute such a combined
|
||||
library, provided that the separate distribution of the work based on
|
||||
the Library and of the other library facilities is otherwise
|
||||
permitted, and provided that you do these two things:
|
||||
|
||||
a) Accompany the combined library with a copy of the same work
|
||||
based on the Library, uncombined with any other library
|
||||
facilities. This must be distributed under the terms of the
|
||||
Sections above.
|
||||
|
||||
b) Give prominent notice with the combined library of the fact
|
||||
that part of it is a work based on the Library, and explaining
|
||||
where to find the accompanying uncombined form of the same work.
|
||||
|
||||
8. You may not copy, modify, sublicense, link with, or distribute
|
||||
the Library except as expressly provided under this License. Any
|
||||
attempt otherwise to copy, modify, sublicense, link with, or
|
||||
distribute the Library is void, and will automatically terminate your
|
||||
rights under this License. However, parties who have received copies,
|
||||
or rights, from you under this License will not have their licenses
|
||||
terminated so long as such parties remain in full compliance.
|
||||
|
||||
9. You are not required to accept this License, since you have not
|
||||
signed it. However, nothing else grants you permission to modify or
|
||||
distribute the Library or its derivative works. These actions are
|
||||
prohibited by law if you do not accept this License. Therefore, by
|
||||
modifying or distributing the Library (or any work based on the
|
||||
Library), you indicate your acceptance of this License to do so, and
|
||||
all its terms and conditions for copying, distributing or modifying
|
||||
the Library or works based on it.
|
||||
|
||||
10. Each time you redistribute the Library (or any work based on the
|
||||
Library), the recipient automatically receives a license from the
|
||||
original licensor to copy, distribute, link with or modify the Library
|
||||
subject to these terms and conditions. You may not impose any further
|
||||
restrictions on the recipients' exercise of the rights granted herein.
|
||||
You are not responsible for enforcing compliance by third parties with
|
||||
this License.
|
||||
^L
|
||||
11. If, as a consequence of a court judgment or allegation of patent
|
||||
infringement or for any other reason (not limited to patent issues),
|
||||
conditions are imposed on you (whether by court order, agreement or
|
||||
otherwise) that contradict the conditions of this License, they do not
|
||||
excuse you from the conditions of this License. If you cannot
|
||||
distribute so as to satisfy simultaneously your obligations under this
|
||||
License and any other pertinent obligations, then as a consequence you
|
||||
may not distribute the Library at all. For example, if a patent
|
||||
license would not permit royalty-free redistribution of the Library by
|
||||
all those who receive copies directly or indirectly through you, then
|
||||
the only way you could satisfy both it and this License would be to
|
||||
refrain entirely from distribution of the Library.
|
||||
|
||||
If any portion of this section is held invalid or unenforceable under
|
||||
any particular circumstance, the balance of the section is intended to
|
||||
apply, and the section as a whole is intended to apply in other
|
||||
circumstances.
|
||||
|
||||
It is not the purpose of this section to induce you to infringe any
|
||||
patents or other property right claims or to contest validity of any
|
||||
such claims; this section has the sole purpose of protecting the
|
||||
integrity of the free software distribution system which is
|
||||
implemented by public license practices. Many people have made
|
||||
generous contributions to the wide range of software distributed
|
||||
through that system in reliance on consistent application of that
|
||||
system; it is up to the author/donor to decide if he or she is willing
|
||||
to distribute software through any other system and a licensee cannot
|
||||
impose that choice.
|
||||
|
||||
This section is intended to make thoroughly clear what is believed to
|
||||
be a consequence of the rest of this License.
|
||||
|
||||
12. If the distribution and/or use of the Library is restricted in
|
||||
certain countries either by patents or by copyrighted interfaces, the
|
||||
original copyright holder who places the Library under this License
|
||||
may add an explicit geographical distribution limitation excluding those
|
||||
countries, so that distribution is permitted only in or among
|
||||
countries not thus excluded. In such case, this License incorporates
|
||||
the limitation as if written in the body of this License.
|
||||
|
||||
13. The Free Software Foundation may publish revised and/or new
|
||||
versions of the Lesser General Public License from time to time.
|
||||
Such new versions will be similar in spirit to the present version,
|
||||
but may differ in detail to address new problems or concerns.
|
||||
|
||||
Each version is given a distinguishing version number. If the Library
|
||||
specifies a version number of this License which applies to it and
|
||||
"any later version", you have the option of following the terms and
|
||||
conditions either of that version or of any later version published by
|
||||
the Free Software Foundation. If the Library does not specify a
|
||||
license version number, you may choose any version ever published by
|
||||
the Free Software Foundation.
|
||||
^L
|
||||
14. If you wish to incorporate parts of the Library into other free
|
||||
programs whose distribution conditions are incompatible with these,
|
||||
write to the author to ask for permission. For software which is
|
||||
copyrighted by the Free Software Foundation, write to the Free
|
||||
Software Foundation; we sometimes make exceptions for this. Our
|
||||
decision will be guided by the two goals of preserving the free status
|
||||
of all derivatives of our free software and of promoting the sharing
|
||||
and reuse of software generally.
|
||||
|
||||
NO WARRANTY
|
||||
|
||||
15. BECAUSE THE LIBRARY IS LICENSED FREE OF CHARGE, THERE IS NO
|
||||
WARRANTY FOR THE LIBRARY, TO THE EXTENT PERMITTED BY APPLICABLE LAW.
|
||||
EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR
|
||||
OTHER PARTIES PROVIDE THE LIBRARY "AS IS" WITHOUT WARRANTY OF ANY
|
||||
KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
|
||||
PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE
|
||||
LIBRARY IS WITH YOU. SHOULD THE LIBRARY PROVE DEFECTIVE, YOU ASSUME
|
||||
THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
|
||||
|
||||
16. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN
|
||||
WRITING WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY
|
||||
AND/OR REDISTRIBUTE THE LIBRARY AS PERMITTED ABOVE, BE LIABLE TO YOU
|
||||
FOR DAMAGES, INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR
|
||||
CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE THE
|
||||
LIBRARY (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING
|
||||
RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A
|
||||
FAILURE OF THE LIBRARY TO OPERATE WITH ANY OTHER SOFTWARE), EVEN IF
|
||||
SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH
|
||||
DAMAGES.
|
||||
|
||||
END OF TERMS AND CONDITIONS
|
||||
^L
|
||||
How to Apply These Terms to Your New Libraries
|
||||
|
||||
If you develop a new library, and you want it to be of the greatest
|
||||
possible use to the public, we recommend making it free software that
|
||||
everyone can redistribute and change. You can do so by permitting
|
||||
redistribution under these terms (or, alternatively, under the terms
|
||||
of the ordinary General Public License).
|
||||
|
||||
To apply these terms, attach the following notices to the library.
|
||||
It is safest to attach them to the start of each source file to most
|
||||
effectively convey the exclusion of warranty; and each file should
|
||||
have at least the "copyright" line and a pointer to where the full
|
||||
notice is found.
|
||||
|
||||
|
||||
<one line to give the library's name and a brief idea of what it
|
||||
does.>
|
||||
Copyright (C) <year> <name of author>
|
||||
|
||||
This library is free software; you can redistribute it and/or
|
||||
modify it under the terms of the GNU Lesser General Public
|
||||
License as published by the Free Software Foundation; either
|
||||
version 2 of the License, or (at your option) any later version.
|
||||
|
||||
This library is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
Lesser General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Lesser General Public
|
||||
License along with this library; if not, write to the Free Software
|
||||
Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
||||
|
||||
Also add information on how to contact you by electronic and paper
|
||||
mail.
|
||||
|
||||
You should also get your employer (if you work as a programmer) or
|
||||
your
|
||||
school, if any, to sign a "copyright disclaimer" for the library, if
|
||||
necessary. Here is a sample; alter the names:
|
||||
|
||||
Yoyodyne, Inc., hereby disclaims all copyright interest in the
|
||||
library `Frob' (a library for tweaking knobs) written by James
|
||||
Random Hacker.
|
||||
|
||||
<signature of Ty Coon>, 1 April 1990
|
||||
Ty Coon, President of Vice
|
||||
|
||||
That's all there is to it!
|
||||
|
||||
|
||||
|
|
@ -1,470 +0,0 @@
|
|||
MOZILLA PUBLIC LICENSE
|
||||
Version 1.1
|
||||
|
||||
---------------
|
||||
|
||||
1. Definitions.
|
||||
|
||||
1.0.1. "Commercial Use" means distribution or otherwise making the
|
||||
Covered Code available to a third party.
|
||||
|
||||
1.1. "Contributor" means each entity that creates or contributes to
|
||||
the creation of Modifications.
|
||||
|
||||
1.2. "Contributor Version" means the combination of the Original
|
||||
Code, prior Modifications used by a Contributor, and the Modifications
|
||||
made by that particular Contributor.
|
||||
|
||||
1.3. "Covered Code" means the Original Code or Modifications or the
|
||||
combination of the Original Code and Modifications, in each case
|
||||
including portions thereof.
|
||||
|
||||
1.4. "Electronic Distribution Mechanism" means a mechanism generally
|
||||
accepted in the software development community for the electronic
|
||||
transfer of data.
|
||||
|
||||
1.5. "Executable" means Covered Code in any form other than Source
|
||||
Code.
|
||||
|
||||
1.6. "Initial Developer" means the individual or entity identified
|
||||
as the Initial Developer in the Source Code notice required by Exhibit
|
||||
A.
|
||||
|
||||
1.7. "Larger Work" means a work which combines Covered Code or
|
||||
portions thereof with code not governed by the terms of this License.
|
||||
|
||||
1.8. "License" means this document.
|
||||
|
||||
1.8.1. "Licensable" means having the right to grant, to the maximum
|
||||
extent possible, whether at the time of the initial grant or
|
||||
subsequently acquired, any and all of the rights conveyed herein.
|
||||
|
||||
1.9. "Modifications" means any addition to or deletion from the
|
||||
substance or structure of either the Original Code or any previous
|
||||
Modifications. When Covered Code is released as a series of files, a
|
||||
Modification is:
|
||||
A. Any addition to or deletion from the contents of a file
|
||||
containing Original Code or previous Modifications.
|
||||
|
||||
B. Any new file that contains any part of the Original Code or
|
||||
previous Modifications.
|
||||
|
||||
1.10. "Original Code" means Source Code of computer software code
|
||||
which is described in the Source Code notice required by Exhibit A as
|
||||
Original Code, and which, at the time of its release under this
|
||||
License is not already Covered Code governed by this License.
|
||||
|
||||
1.10.1. "Patent Claims" means any patent claim(s), now owned or
|
||||
hereafter acquired, including without limitation, method, process,
|
||||
and apparatus claims, in any patent Licensable by grantor.
|
||||
|
||||
1.11. "Source Code" means the preferred form of the Covered Code for
|
||||
making modifications to it, including all modules it contains, plus
|
||||
any associated interface definition files, scripts used to control
|
||||
compilation and installation of an Executable, or source code
|
||||
differential comparisons against either the Original Code or another
|
||||
well known, available Covered Code of the Contributor's choice. The
|
||||
Source Code can be in a compressed or archival form, provided the
|
||||
appropriate decompression or de-archiving software is widely available
|
||||
for no charge.
|
||||
|
||||
1.12. "You" (or "Your") means an individual or a legal entity
|
||||
exercising rights under, and complying with all of the terms of, this
|
||||
License or a future version of this License issued under Section 6.1.
|
||||
For legal entities, "You" includes any entity which controls, is
|
||||
controlled by, or is under common control with You. For purposes of
|
||||
this definition, "control" means (a) the power, direct or indirect,
|
||||
to cause the direction or management of such entity, whether by
|
||||
contract or otherwise, or (b) ownership of more than fifty percent
|
||||
(50%) of the outstanding shares or beneficial ownership of such
|
||||
entity.
|
||||
|
||||
2. Source Code License.
|
||||
|
||||
2.1. The Initial Developer Grant.
|
||||
The Initial Developer hereby grants You a world-wide, royalty-free,
|
||||
non-exclusive license, subject to third party intellectual property
|
||||
claims:
|
||||
(a) under intellectual property rights (other than patent or
|
||||
trademark) Licensable by Initial Developer to use, reproduce,
|
||||
modify, display, perform, sublicense and distribute the Original
|
||||
Code (or portions thereof) with or without Modifications, and/or
|
||||
as part of a Larger Work; and
|
||||
|
||||
(b) under Patents Claims infringed by the making, using or
|
||||
selling of Original Code, to make, have made, use, practice,
|
||||
sell, and offer for sale, and/or otherwise dispose of the
|
||||
Original Code (or portions thereof).
|
||||
|
||||
(c) the licenses granted in this Section 2.1(a) and (b) are
|
||||
effective on the date Initial Developer first distributes
|
||||
Original Code under the terms of this License.
|
||||
|
||||
(d) Notwithstanding Section 2.1(b) above, no patent license is
|
||||
granted: 1) for code that You delete from the Original Code; 2)
|
||||
separate from the Original Code; or 3) for infringements caused
|
||||
by: i) the modification of the Original Code or ii) the
|
||||
combination of the Original Code with other software or devices.
|
||||
|
||||
2.2. Contributor Grant.
|
||||
Subject to third party intellectual property claims, each Contributor
|
||||
hereby grants You a world-wide, royalty-free, non-exclusive license
|
||||
|
||||
(a) under intellectual property rights (other than patent or
|
||||
trademark) Licensable by Contributor, to use, reproduce, modify,
|
||||
display, perform, sublicense and distribute the Modifications
|
||||
created by such Contributor (or portions thereof) either on an
|
||||
unmodified basis, with other Modifications, as Covered Code
|
||||
and/or as part of a Larger Work; and
|
||||
|
||||
(b) under Patent Claims infringed by the making, using, or
|
||||
selling of Modifications made by that Contributor either alone
|
||||
and/or in combination with its Contributor Version (or portions
|
||||
of such combination), to make, use, sell, offer for sale, have
|
||||
made, and/or otherwise dispose of: 1) Modifications made by that
|
||||
Contributor (or portions thereof); and 2) the combination of
|
||||
Modifications made by that Contributor with its Contributor
|
||||
Version (or portions of such combination).
|
||||
|
||||
(c) the licenses granted in Sections 2.2(a) and 2.2(b) are
|
||||
effective on the date Contributor first makes Commercial Use of
|
||||
the Covered Code.
|
||||
|
||||
(d) Notwithstanding Section 2.2(b) above, no patent license is
|
||||
granted: 1) for any code that Contributor has deleted from the
|
||||
Contributor Version; 2) separate from the Contributor Version;
|
||||
3) for infringements caused by: i) third party modifications of
|
||||
Contributor Version or ii) the combination of Modifications made
|
||||
by that Contributor with other software (except as part of the
|
||||
Contributor Version) or other devices; or 4) under Patent Claims
|
||||
infringed by Covered Code in the absence of Modifications made by
|
||||
that Contributor.
|
||||
|
||||
3. Distribution Obligations.
|
||||
|
||||
3.1. Application of License.
|
||||
The Modifications which You create or to which You contribute are
|
||||
governed by the terms of this License, including without limitation
|
||||
Section 2.2. The Source Code version of Covered Code may be
|
||||
distributed only under the terms of this License or a future version
|
||||
of this License released under Section 6.1, and You must include a
|
||||
copy of this License with every copy of the Source Code You
|
||||
distribute. You may not offer or impose any terms on any Source Code
|
||||
version that alters or restricts the applicable version of this
|
||||
License or the recipients' rights hereunder. However, You may include
|
||||
an additional document offering the additional rights described in
|
||||
Section 3.5.
|
||||
|
||||
3.2. Availability of Source Code.
|
||||
Any Modification which You create or to which You contribute must be
|
||||
made available in Source Code form under the terms of this License
|
||||
either on the same media as an Executable version or via an accepted
|
||||
Electronic Distribution Mechanism to anyone to whom you made an
|
||||
Executable version available; and if made available via Electronic
|
||||
Distribution Mechanism, must remain available for at least twelve (12)
|
||||
months after the date it initially became available, or at least six
|
||||
(6) months after a subsequent version of that particular Modification
|
||||
has been made available to such recipients. You are responsible for
|
||||
ensuring that the Source Code version remains available even if the
|
||||
Electronic Distribution Mechanism is maintained by a third party.
|
||||
|
||||
3.3. Description of Modifications.
|
||||
You must cause all Covered Code to which You contribute to contain a
|
||||
file documenting the changes You made to create that Covered Code and
|
||||
the date of any change. You must include a prominent statement that
|
||||
the Modification is derived, directly or indirectly, from Original
|
||||
Code provided by the Initial Developer and including the name of the
|
||||
Initial Developer in (a) the Source Code, and (b) in any notice in an
|
||||
Executable version or related documentation in which You describe the
|
||||
origin or ownership of the Covered Code.
|
||||
|
||||
3.4. Intellectual Property Matters
|
||||
(a) Third Party Claims.
|
||||
If Contributor has knowledge that a license under a third party's
|
||||
intellectual property rights is required to exercise the rights
|
||||
granted by such Contributor under Sections 2.1 or 2.2,
|
||||
Contributor must include a text file with the Source Code
|
||||
distribution titled "LEGAL" which describes the claim and the
|
||||
party making the claim in sufficient detail that a recipient will
|
||||
know whom to contact. If Contributor obtains such knowledge after
|
||||
the Modification is made available as described in Section 3.2,
|
||||
Contributor shall promptly modify the LEGAL file in all copies
|
||||
Contributor makes available thereafter and shall take other steps
|
||||
(such as notifying appropriate mailing lists or newsgroups)
|
||||
reasonably calculated to inform those who received the Covered
|
||||
Code that new knowledge has been obtained.
|
||||
|
||||
(b) Contributor APIs.
|
||||
If Contributor's Modifications include an application programming
|
||||
interface and Contributor has knowledge of patent licenses which
|
||||
are reasonably necessary to implement that API, Contributor must
|
||||
also include this information in the LEGAL file.
|
||||
|
||||
(c) Representations.
|
||||
Contributor represents that, except as disclosed pursuant to
|
||||
Section 3.4(a) above, Contributor believes that Contributor's
|
||||
Modifications are Contributor's original creation(s) and/or
|
||||
Contributor has sufficient rights to grant the rights conveyed by
|
||||
this License.
|
||||
|
||||
3.5. Required Notices.
|
||||
You must duplicate the notice in Exhibit A in each file of the Source
|
||||
Code. If it is not possible to put such notice in a particular Source
|
||||
Code file due to its structure, then You must include such notice in a
|
||||
location (such as a relevant directory) where a user would be likely
|
||||
to look for such a notice. If You created one or more Modification(s)
|
||||
You may add your name as a Contributor to the notice described in
|
||||
Exhibit A. You must also duplicate this License in any documentation
|
||||
for the Source Code where You describe recipients' rights or ownership
|
||||
rights relating to Covered Code. You may choose to offer, and to
|
||||
charge a fee for, warranty, support, indemnity or liability
|
||||
obligations to one or more recipients of Covered Code. However, You
|
||||
may do so only on Your own behalf, and not on behalf of the Initial
|
||||
Developer or any Contributor. You must make it absolutely clear than
|
||||
any such warranty, support, indemnity or liability obligation is
|
||||
offered by You alone, and You hereby agree to indemnify the Initial
|
||||
Developer and every Contributor for any liability incurred by the
|
||||
Initial Developer or such Contributor as a result of warranty,
|
||||
support, indemnity or liability terms You offer.
|
||||
|
||||
3.6. Distribution of Executable Versions.
|
||||
You may distribute Covered Code in Executable form only if the
|
||||
requirements of Section 3.1-3.5 have been met for that Covered Code,
|
||||
and if You include a notice stating that the Source Code version of
|
||||
the Covered Code is available under the terms of this License,
|
||||
including a description of how and where You have fulfilled the
|
||||
obligations of Section 3.2. The notice must be conspicuously included
|
||||
in any notice in an Executable version, related documentation or
|
||||
collateral in which You describe recipients' rights relating to the
|
||||
Covered Code. You may distribute the Executable version of Covered
|
||||
Code or ownership rights under a license of Your choice, which may
|
||||
contain terms different from this License, provided that You are in
|
||||
compliance with the terms of this License and that the license for the
|
||||
Executable version does not attempt to limit or alter the recipient's
|
||||
rights in the Source Code version from the rights set forth in this
|
||||
License. If You distribute the Executable version under a different
|
||||
license You must make it absolutely clear that any terms which differ
|
||||
from this License are offered by You alone, not by the Initial
|
||||
Developer or any Contributor. You hereby agree to indemnify the
|
||||
Initial Developer and every Contributor for any liability incurred by
|
||||
the Initial Developer or such Contributor as a result of any such
|
||||
terms You offer.
|
||||
|
||||
3.7. Larger Works.
|
||||
You may create a Larger Work by combining Covered Code with other code
|
||||
not governed by the terms of this License and distribute the Larger
|
||||
Work as a single product. In such a case, You must make sure the
|
||||
requirements of this License are fulfilled for the Covered Code.
|
||||
|
||||
4. Inability to Comply Due to Statute or Regulation.
|
||||
|
||||
If it is impossible for You to comply with any of the terms of this
|
||||
License with respect to some or all of the Covered Code due to
|
||||
statute, judicial order, or regulation then You must: (a) comply with
|
||||
the terms of this License to the maximum extent possible; and (b)
|
||||
describe the limitations and the code they affect. Such description
|
||||
must be included in the LEGAL file described in Section 3.4 and must
|
||||
be included with all distributions of the Source Code. Except to the
|
||||
extent prohibited by statute or regulation, such description must be
|
||||
sufficiently detailed for a recipient of ordinary skill to be able to
|
||||
understand it.
|
||||
|
||||
5. Application of this License.
|
||||
|
||||
This License applies to code to which the Initial Developer has
|
||||
attached the notice in Exhibit A and to related Covered Code.
|
||||
|
||||
6. Versions of the License.
|
||||
|
||||
6.1. New Versions.
|
||||
Netscape Communications Corporation ("Netscape") may publish revised
|
||||
and/or new versions of the License from time to time. Each version
|
||||
will be given a distinguishing version number.
|
||||
|
||||
6.2. Effect of New Versions.
|
||||
Once Covered Code has been published under a particular version of the
|
||||
License, You may always continue to use it under the terms of that
|
||||
version. You may also choose to use such Covered Code under the terms
|
||||
of any subsequent version of the License published by Netscape. No one
|
||||
other than Netscape has the right to modify the terms applicable to
|
||||
Covered Code created under this License.
|
||||
|
||||
6.3. Derivative Works.
|
||||
If You create or use a modified version of this License (which you may
|
||||
only do in order to apply it to code which is not already Covered Code
|
||||
governed by this License), You must (a) rename Your license so that
|
||||
the phrases "Mozilla", "MOZILLAPL", "MOZPL", "Netscape",
|
||||
"MPL", "NPL" or any confusingly similar phrase do not appear in your
|
||||
license (except to note that your license differs from this License)
|
||||
and (b) otherwise make it clear that Your version of the license
|
||||
contains terms which differ from the Mozilla Public License and
|
||||
Netscape Public License. (Filling in the name of the Initial
|
||||
Developer, Original Code or Contributor in the notice described in
|
||||
Exhibit A shall not of themselves be deemed to be modifications of
|
||||
this License.)
|
||||
|
||||
7. DISCLAIMER OF WARRANTY.
|
||||
|
||||
COVERED CODE IS PROVIDED UNDER THIS LICENSE ON AN "AS IS" BASIS,
|
||||
WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING,
|
||||
WITHOUT LIMITATION, WARRANTIES THAT THE COVERED CODE IS FREE OF
|
||||
DEFECTS, MERCHANTABLE, FIT FOR A PARTICULAR PURPOSE OR NON-INFRINGING.
|
||||
THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE COVERED CODE
|
||||
IS WITH YOU. SHOULD ANY COVERED CODE PROVE DEFECTIVE IN ANY RESPECT,
|
||||
YOU (NOT THE INITIAL DEVELOPER OR ANY OTHER CONTRIBUTOR) ASSUME THE
|
||||
COST OF ANY NECESSARY SERVICING, REPAIR OR CORRECTION. THIS DISCLAIMER
|
||||
OF WARRANTY CONSTITUTES AN ESSENTIAL PART OF THIS LICENSE. NO USE OF
|
||||
ANY COVERED CODE IS AUTHORIZED HEREUNDER EXCEPT UNDER THIS DISCLAIMER.
|
||||
|
||||
8. TERMINATION.
|
||||
|
||||
8.1. This License and the rights granted hereunder will terminate
|
||||
automatically if You fail to comply with terms herein and fail to cure
|
||||
such breach within 30 days of becoming aware of the breach. All
|
||||
sublicenses to the Covered Code which are properly granted shall
|
||||
survive any termination of this License. Provisions which, by their
|
||||
nature, must remain in effect beyond the termination of this License
|
||||
shall survive.
|
||||
|
||||
8.2. If You initiate litigation by asserting a patent infringement
|
||||
claim (excluding declatory judgment actions) against Initial Developer
|
||||
or a Contributor (the Initial Developer or Contributor against whom
|
||||
You file such action is referred to as "Participant") alleging that:
|
||||
|
||||
(a) such Participant's Contributor Version directly or indirectly
|
||||
infringes any patent, then any and all rights granted by such
|
||||
Participant to You under Sections 2.1 and/or 2.2 of this License
|
||||
shall, upon 60 days notice from Participant terminate prospectively,
|
||||
unless if within 60 days after receipt of notice You either: (i)
|
||||
agree in writing to pay Participant a mutually agreeable reasonable
|
||||
royalty for Your past and future use of Modifications made by such
|
||||
Participant, or (ii) withdraw Your litigation claim with respect to
|
||||
the Contributor Version against such Participant. If within 60 days
|
||||
of notice, a reasonable royalty and payment arrangement are not
|
||||
mutually agreed upon in writing by the parties or the litigation claim
|
||||
is not withdrawn, the rights granted by Participant to You under
|
||||
Sections 2.1 and/or 2.2 automatically terminate at the expiration of
|
||||
the 60 day notice period specified above.
|
||||
|
||||
(b) any software, hardware, or device, other than such Participant's
|
||||
Contributor Version, directly or indirectly infringes any patent, then
|
||||
any rights granted to You by such Participant under Sections 2.1(b)
|
||||
and 2.2(b) are revoked effective as of the date You first made, used,
|
||||
sold, distributed, or had made, Modifications made by that
|
||||
Participant.
|
||||
|
||||
8.3. If You assert a patent infringement claim against Participant
|
||||
alleging that such Participant's Contributor Version directly or
|
||||
indirectly infringes any patent where such claim is resolved (such as
|
||||
by license or settlement) prior to the initiation of patent
|
||||
infringement litigation, then the reasonable value of the licenses
|
||||
granted by such Participant under Sections 2.1 or 2.2 shall be taken
|
||||
into account in determining the amount or value of any payment or
|
||||
license.
|
||||
|
||||
8.4. In the event of termination under Sections 8.1 or 8.2 above,
|
||||
all end user license agreements (excluding distributors and resellers)
|
||||
which have been validly granted by You or any distributor hereunder
|
||||
prior to termination shall survive termination.
|
||||
|
||||
9. LIMITATION OF LIABILITY.
|
||||
|
||||
UNDER NO CIRCUMSTANCES AND UNDER NO LEGAL THEORY, WHETHER TORT
|
||||
(INCLUDING NEGLIGENCE), CONTRACT, OR OTHERWISE, SHALL YOU, THE INITIAL
|
||||
DEVELOPER, ANY OTHER CONTRIBUTOR, OR ANY DISTRIBUTOR OF COVERED CODE,
|
||||
OR ANY SUPPLIER OF ANY OF SUCH PARTIES, BE LIABLE TO ANY PERSON FOR
|
||||
ANY INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES OF ANY
|
||||
CHARACTER INCLUDING, WITHOUT LIMITATION, DAMAGES FOR LOSS OF GOODWILL,
|
||||
WORK STOPPAGE, COMPUTER FAILURE OR MALFUNCTION, OR ANY AND ALL OTHER
|
||||
COMMERCIAL DAMAGES OR LOSSES, EVEN IF SUCH PARTY SHALL HAVE BEEN
|
||||
INFORMED OF THE POSSIBILITY OF SUCH DAMAGES. THIS LIMITATION OF
|
||||
LIABILITY SHALL NOT APPLY TO LIABILITY FOR DEATH OR PERSONAL INJURY
|
||||
RESULTING FROM SUCH PARTY'S NEGLIGENCE TO THE EXTENT APPLICABLE LAW
|
||||
PROHIBITS SUCH LIMITATION. SOME JURISDICTIONS DO NOT ALLOW THE
|
||||
EXCLUSION OR LIMITATION OF INCIDENTAL OR CONSEQUENTIAL DAMAGES, SO
|
||||
THIS EXCLUSION AND LIMITATION MAY NOT APPLY TO YOU.
|
||||
|
||||
10. U.S. GOVERNMENT END USERS.
|
||||
|
||||
The Covered Code is a "commercial item," as that term is defined in
|
||||
48 C.F.R. 2.101 (Oct. 1995), consisting of "commercial computer
|
||||
software" and "commercial computer software documentation," as such
|
||||
terms are used in 48 C.F.R. 12.212 (Sept. 1995). Consistent with 48
|
||||
C.F.R. 12.212 and 48 C.F.R. 227.7202-1 through 227.7202-4 (June 1995),
|
||||
all U.S. Government End Users acquire Covered Code with only those
|
||||
rights set forth herein.
|
||||
|
||||
11. MISCELLANEOUS.
|
||||
|
||||
This License represents the complete agreement concerning subject
|
||||
matter hereof. If any provision of this License is held to be
|
||||
unenforceable, such provision shall be reformed only to the extent
|
||||
necessary to make it enforceable. This License shall be governed by
|
||||
California law provisions (except to the extent applicable law, if
|
||||
any, provides otherwise), excluding its conflict-of-law provisions.
|
||||
With respect to disputes in which at least one party is a citizen of,
|
||||
or an entity chartered or registered to do business in the United
|
||||
States of America, any litigation relating to this License shall be
|
||||
subject to the jurisdiction of the Federal Courts of the Northern
|
||||
District of California, with venue lying in Santa Clara County,
|
||||
California, with the losing party responsible for costs, including
|
||||
without limitation, court costs and reasonable attorneys' fees and
|
||||
expenses. The application of the United Nations Convention on
|
||||
Contracts for the International Sale of Goods is expressly excluded.
|
||||
Any law or regulation which provides that the language of a contract
|
||||
shall be construed against the drafter shall not apply to this
|
||||
License.
|
||||
|
||||
12. RESPONSIBILITY FOR CLAIMS.
|
||||
|
||||
As between Initial Developer and the Contributors, each party is
|
||||
responsible for claims and damages arising, directly or indirectly,
|
||||
out of its utilization of rights under this License and You agree to
|
||||
work with Initial Developer and Contributors to distribute such
|
||||
responsibility on an equitable basis. Nothing herein is intended or
|
||||
shall be deemed to constitute any admission of liability.
|
||||
|
||||
13. MULTIPLE-LICENSED CODE.
|
||||
|
||||
Initial Developer may designate portions of the Covered Code as
|
||||
"Multiple-Licensed". "Multiple-Licensed" means that the Initial
|
||||
Developer permits you to utilize portions of the Covered Code under
|
||||
Your choice of the NPL or the alternative licenses, if any, specified
|
||||
by the Initial Developer in the file described in Exhibit A.
|
||||
|
||||
EXHIBIT A -Mozilla Public License.
|
||||
|
||||
``The contents of this file are subject to the Mozilla Public License
|
||||
Version 1.1 (the "License"); you may not use this file except in
|
||||
compliance with the License. You may obtain a copy of the License at
|
||||
http://www.mozilla.org/MPL/
|
||||
|
||||
Software distributed under the License is distributed on an "AS IS"
|
||||
basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See the
|
||||
License for the specific language governing rights and limitations
|
||||
under the License.
|
||||
|
||||
The Original Code is ______________________________________.
|
||||
|
||||
The Initial Developer of the Original Code is ________________________.
|
||||
Portions created by ______________________ are Copyright (C) ______
|
||||
_______________________. All Rights Reserved.
|
||||
|
||||
Contributor(s): ______________________________________.
|
||||
|
||||
Alternatively, the contents of this file may be used under the terms
|
||||
of the _____ license (the "[___] License"), in which case the
|
||||
provisions of [______] License are applicable instead of those
|
||||
above. If you wish to allow use of your version of this file only
|
||||
under the terms of the [____] License and not to allow others to use
|
||||
your version of this file under the MPL, indicate your decision by
|
||||
deleting the provisions above and replace them with the notice and
|
||||
other provisions required by the [___] License. If you do not delete
|
||||
the provisions above, a recipient may use your version of this file
|
||||
under either the MPL or the [___] License."
|
||||
|
||||
[NOTE: The text of this Exhibit A may differ slightly from the text of
|
||||
the notices in the Source Code files of the Original Code. You should
|
||||
use the text of this Exhibit A rather than the text found in the
|
||||
Original Code Source Code for Your Modifications.]
|
||||
|
||||
|
|
@ -1,106 +0,0 @@
|
|||
2014-09-18 Hyphen 2.8.8:
|
||||
- remove last coverity warning, 0 remaining
|
||||
|
||||
2014-06-27 Hyphen 2.8.7:
|
||||
- various clang scan-build warning fixes
|
||||
|
||||
2012-09-13 Hyphen 2.8.6:
|
||||
- righthyphenmin fix for 3-byte or more UTF-8
|
||||
multibyte characters by Steven Dickson
|
||||
- fix for fdo#43931 (removing hard hyphen hyphenation for LibreOffice)
|
||||
|
||||
2012-07-12 Hyphen 2.8.5:
|
||||
- fix short alloc
|
||||
|
||||
2012-06-29 Hyphen 2.8.4:
|
||||
- coverity warnings
|
||||
|
||||
2011-10-10 Hyphen 2.8.3:
|
||||
- fix NOHYPHEN
|
||||
- fix unbalanced hyphenation of LibreOffice/OOo
|
||||
- set default COMPOUNDHYPHENMIN=3 at hyphens and apostrophes
|
||||
- fix VERBOSE in hyphen.c
|
||||
- new ./example option: -n to print hyphenation vector
|
||||
|
||||
2011-10-07 Hyphen 2.8.2:
|
||||
- fix for explicite COMPOUNDHYPHENMIN values
|
||||
|
||||
2011-10-06 Hyphen 2.8.1:
|
||||
- force minimal lefthyphenmin and righthyphenmin values of the dictionary
|
||||
(eg. righthyphenmin=3 of English dictionaries in LibreOffice/OOo,
|
||||
also the original TeX hyphenation patterns are correct only with this
|
||||
righthyphenmin value).
|
||||
|
||||
2011-10-04 Hyphen 2.8:
|
||||
- Ignore leading and ending numbers (eg. manual/field based indexes
|
||||
in LibreOffice/OOo)
|
||||
|
||||
- Fix LibreOffice/OpenOffice.org hyphenation errors at apostrophes and
|
||||
hyphens, n-dashes with default NOHYPHEN separators.
|
||||
Eg. *o'c=lock -> o'clock.
|
||||
|
||||
2010-12-01 Hyphen 2.7.1 bug fix release
|
||||
|
||||
2010-11-27 Hyphen 2.7 release:
|
||||
- The new hyphenation problem of OpenOffice.org 3.2, related to its
|
||||
modified word breaking of words with hyphen characters, can be fixed
|
||||
with the new NOHYPHEN feature. Also it's possible to solve the similar old
|
||||
problem with apostrophes. More information: README.compound.
|
||||
|
||||
- improved English dictionaries
|
||||
|
||||
2010-08-10 Hyphen 2.6 release:
|
||||
- maintainance release, fix all warnings, tidy up
|
||||
make check with VALGRIND=memcheck, etc.
|
||||
|
||||
2010-02-23 Hyphen 2.5 release:
|
||||
- add Unicode ligature support for correct hyphenmin calculation
|
||||
(ff, fi, fl, St, st are 1-character, ffi and ffl are 2-character length for
|
||||
hyphenation)
|
||||
- fix lefthyphenmin calculation for UTF-8 encoded input
|
||||
|
||||
- en_US hyphenation dictionary:
|
||||
- add OpenOffice.org patch to fix apostrophe handling
|
||||
- add correct hyphenation for words with Unicode f-ligatures
|
||||
(NOTE: hyphenation within ligatures is not supported yet
|
||||
because of an implementation problem of OpenOffice.org,
|
||||
see OOo issue 71608.)
|
||||
|
||||
- small patches from OpenOffice.org
|
||||
|
||||
2008-05-01 Hyphen 2.4 release:
|
||||
- compound word hyphenation support by recursive pattern matching
|
||||
based on two hyphenation pattern sets, see README.compound.
|
||||
Especially useful for languages with arbitrary number of compounds (Danish,
|
||||
Dutch, Finnish, German, Hungarian, Icelandic, Norwegian, Swedish etc.).
|
||||
|
||||
- new dictionary parameters (minimal character numbers for hyph. distances):
|
||||
LEFTHYPHENMIN: minimal hyphenation distance from the left end of the word
|
||||
RIGHTHYPHENMIN: minimal hyphenation distance from the right end of the word
|
||||
COMPOUNDLEFTHYPHENMIN: min. hyph. dist. from the left compound word boundary
|
||||
COMPOUNDRIGHTHYPHENMIN: min. hyph. dist. from the right comp. word boundary
|
||||
|
||||
- new API function: hnj_hyphen_hyphenate3() (like hyphenate2(), but
|
||||
with hyphenmin options)
|
||||
|
||||
en_US hyphenation patterns:
|
||||
|
||||
- extended hyph_en_US.dic with TugBoat hyphenation log (fix thousand
|
||||
incompletely or badly hyphenated words, for example acad-e-my, acro-nym,
|
||||
acryl-amide, adren-a-line, aero-space, am-phet-a-mine, anom-aly etc.)
|
||||
|
||||
- fixed hyph_en_US.dic: set the right default hyphenation distance of
|
||||
the original TeX hyphenation patterns:
|
||||
LEFTHYPHENMIN 2
|
||||
RIGHTHYPHENMIN 3 (not 2!)
|
||||
It is not only a typographical issue. It seems, TeX hyphenation
|
||||
patterns are right only with these settings, for example,
|
||||
the bad "anoma-ly" is restricted in TeX only by the default
|
||||
\righthyphenmin=3 (but not restricted in OpenOffice.org, until now).
|
||||
|
||||
- documentation (README_hyph_en_US.dic)
|
||||
|
||||
- fixes for automake configuration, compiling and checking, see ChangeLog
|
||||
|
||||
2008-02-19: Hyphen 2.3.1 release:
|
||||
- fix obsolete API function hnj_hyphen_hyphenate()
|
||||
|
|
@ -1,134 +0,0 @@
|
|||
Hyphen - hyphenation library to use converted TeX hyphenation patterns
|
||||
|
||||
(C) 1998 Raph Levien
|
||||
(C) 2001 ALTLinux, Moscow
|
||||
(C) 2006, 2007, 2008, 2010, 2011 László Németh
|
||||
|
||||
This was part of libHnj library by Raph Levien.
|
||||
|
||||
Peter Novodvorsky from ALTLinux cut hyphenation part from libHnj
|
||||
to use it in OpenOffice.org.
|
||||
|
||||
Compound word and non-standard hyphenation support by László Németh.
|
||||
|
||||
License is the original LibHnj license:
|
||||
LibHnj is dual licensed under LGPL and MPL (see also README.libhnj).
|
||||
|
||||
Because LGPL allows GPL relicensing, COPYING contains now
|
||||
LGPL/GPL/MPL tri-license for explicit Mozilla source compatibility.
|
||||
|
||||
Original Libhnj source with OOo's patches are managed by Rene Engelhard
|
||||
and Chris Halls at Debian:
|
||||
|
||||
http://packages.debian.org/stable/libdevel/libhnj-dev
|
||||
and http://packages.debian.org/unstable/source/libhnj
|
||||
|
||||
|
||||
OTHER FILES
|
||||
|
||||
This distribution is the source of the en_US hyphenation patterns
|
||||
"hyph_en_US.dic", too. See README_hyph_en_US.txt.
|
||||
|
||||
Source files of hyph_en_US.dic in the distribution:
|
||||
|
||||
hyphen.tex (en_US hyphenation patterns from plain TeX)
|
||||
|
||||
Source: http://tug.ctan.org/text-archive/macros/plain/base/hyphen.tex
|
||||
|
||||
tbhyphext.tex: hyphenation exception log from TugBoat archive
|
||||
|
||||
Source of the hyphenation exception list:
|
||||
http://www.ctan.org/tex-archive/info/digests/tugboat/tb0hyf.tex
|
||||
|
||||
Generated with the hyphenex script
|
||||
(http://www.ctan.org/tex-archive/info/digests/tugboat/hyphenex.sh)
|
||||
|
||||
sh hyphenex.sh <tb0hyf.tex >tbhyphext.tex
|
||||
|
||||
|
||||
INSTALLATION
|
||||
|
||||
autoreconf -fvi
|
||||
./configure
|
||||
make
|
||||
make install
|
||||
|
||||
UNIT TESTS (WITH VALGRIND DEBUGGER)
|
||||
|
||||
make check
|
||||
VALGRIND=memcheck make check
|
||||
|
||||
USAGE
|
||||
|
||||
./example hyph_en_US.dic mywords.txt
|
||||
|
||||
or (under Linux)
|
||||
|
||||
echo example | ./example hyph_en_US.dic /dev/stdin
|
||||
|
||||
NOTE: In the case of Unicode encoded input, convert your words
|
||||
to lowercase before hyphenation (under UTF-8 console environment):
|
||||
|
||||
cat mywords.txt | awk '{print tolower($0)}' >mywordslow.txt
|
||||
|
||||
BUILD DLL USING CROSS-COMPILATION
|
||||
|
||||
./configure --host i586-mingw32 --prefix=/tmp/hyphen-dll
|
||||
make
|
||||
make install
|
||||
|
||||
DEVELOPMENT
|
||||
|
||||
See README.hyphen for hyphenation algorithm, README.nonstandard
|
||||
and doc/tb87nemeth.pdf for non-standard hyphenation,
|
||||
README.compound for compound word hyphenation, and tests/*.
|
||||
|
||||
Description of the dictionary format:
|
||||
|
||||
First line contains the character encoding (ISO8859-x, UTF-8).
|
||||
|
||||
Possible options in the following lines:
|
||||
|
||||
LEFTHYPHENMIN num minimal hyphenation distance from the left word end
|
||||
RIGHTHYPHENMIN num minimal hyphation distance from the right word end
|
||||
COMPOUNDLEFTHYPHENMIN num min. hyph. dist. from the left compound word boundary
|
||||
COMPOUNDRIGHTHYPHENMIN num min. hyph. dist. from the right comp. word boundary
|
||||
|
||||
hyphenation patterns see README.* files
|
||||
|
||||
NEXTWORD separate the two compound sets (see README.compound)
|
||||
|
||||
Default values:
|
||||
Without explicite declarations, hyphenmin fields of dict struct
|
||||
are zeroes, but in this case the lefthyphenmin and righthyphenmin
|
||||
will be the default 2 under the hyphenation (for backward compatibility).
|
||||
|
||||
Comments
|
||||
|
||||
Use percent sign at the beginning of the lines to add comments to your
|
||||
hpyhenation patterns (after the character encoding in the first line):
|
||||
|
||||
% comment
|
||||
|
||||
*****************************************************************************
|
||||
* Warning! Correct working of Libhnj *needs* prepared hyphenation patterns. *
|
||||
|
||||
For example, generating hyph_en_US.dic from "hyphen.us" TeX patterns:
|
||||
|
||||
perl substrings.pl hyphen.us hyph_en_US.dic ISO8859-1
|
||||
|
||||
or with default LEFTHYPHENMIN and RIGHTHYPHENMIN values:
|
||||
|
||||
perl substrings.pl hyphen.us hyph_en_US.dic ISO8859-1 2 3
|
||||
perl substrings.pl hyphen.gb hyph_en_GB.dic ISO8859-1 3 3
|
||||
****************************************************************************
|
||||
|
||||
OTHERS
|
||||
|
||||
Java hyphenation: Peter B. West (Folio project) implements a hyphenator with
|
||||
non standard hyphenation facilities based on extended Libhnj. The HyFo module
|
||||
is released in binary form as jar files and in source form as zip files.
|
||||
See http://sourceforge.net/project/showfiles.php?group_id=119136
|
||||
|
||||
László Németh
|
||||
<nemeth (at) numbertext (dot) org>
|
||||
|
|
@ -1,87 +0,0 @@
|
|||
New option of Libhyphen 2.7: NOHYPHEN
|
||||
|
||||
Hyphen, apostrophe and other characters may be word boundary characters,
|
||||
but they don't need (extra) hyphenation. With NOHYPHEN option
|
||||
it's possible to hyphenate the words parts correctly.
|
||||
|
||||
Example:
|
||||
|
||||
ISO8859-1
|
||||
NOHYPHEN -,'
|
||||
1-1
|
||||
1'1
|
||||
NEXTLEVEL
|
||||
|
||||
Description:
|
||||
|
||||
1-1 and 1'1 declare hyphen and apostrophe as word boundary characters
|
||||
and NOHYPHEN with the comma separated character (or character sequence)
|
||||
list forbid the (extra) hyphens at the hyphen and apostrophe characters.
|
||||
|
||||
Implicite NOHYPHEN declaration
|
||||
|
||||
Without explicite NEXTLEVEL declaration, Hyphen 2.8 uses the
|
||||
previous settings, plus in UTF-8 encoding, endash (U+2013) and
|
||||
typographical apostrophe (U+2019) are NOHYPHEN characters, too.
|
||||
|
||||
It's possible to enlarge the hyphenation distance from these
|
||||
NOHYPHEN characters by using COMPOUNDLEFTHYPHENMIN and
|
||||
COMPOUNDRIGHTHYPHENMIN attributes.
|
||||
|
||||
Compound word hyphenation
|
||||
|
||||
Hyphen library supports better compound word hyphenation and special
|
||||
rules of compound word hyphenation of German languages and other
|
||||
languages with arbitrary number of compound words. The new options,
|
||||
COMPOUNDLEFTHYPHENMIN and COMPOUNDRIGHTHYPHENMIN help to set the right
|
||||
style for the hyphenation of compound words.
|
||||
|
||||
Algorithm
|
||||
|
||||
The algorithm is an extension of the original pattern based hyphenation
|
||||
algorithm. It uses two hyphenation pattern sets, defined in the same
|
||||
pattern file and separated by the NEXTLEVEL keyword. First pattern
|
||||
set is for hyphenation only at compound word boundaries, the second one
|
||||
is for hyphenation within words or word parts.
|
||||
|
||||
Recursive compound level hyphenation
|
||||
|
||||
The algorithm is recursive: every word parts of a successful
|
||||
first (compound) level hyphenation will be rehyphenated
|
||||
by the same (first) pattern set.
|
||||
|
||||
Finally, when first level hyphenation is not possible, Hyphen uses
|
||||
the second level hyphenation for the word or the word parts.
|
||||
|
||||
Word endings and word parts
|
||||
|
||||
Patterns for word endings (patterns with ellipses) match the
|
||||
word parts, too.
|
||||
|
||||
Options
|
||||
|
||||
COMPOUNDLEFTHYPHENMIN: min. hyph. dist. from the left compound word boundary
|
||||
COMPOUNDRIGHTHYPHENMIN: min. hyph. dist. from the right comp. word boundary
|
||||
NEXTLEVEL: sign second level hyphenation patterns
|
||||
|
||||
Default hyphenmin values
|
||||
|
||||
Default values of COMPOUNDLEFTHYPHENMIN and COMPOUNDRIGHTHYPHENMIN are 0,
|
||||
and 0 under the hyphenation, too. ("0" values of
|
||||
LEFTHYPHENMIN and RIGHTHYPHENMIN mean the default "2" under the hyphenation.)
|
||||
|
||||
Examples
|
||||
|
||||
See tests/compound* test files.
|
||||
|
||||
Preparation of hyphenation patterns
|
||||
|
||||
It hasn't been special pattern generator tool for compound hyphenation
|
||||
patterns, yet. It is possible to use PATGEN to generate both of
|
||||
pattern sets, concatenate it manually and set the requested HYPHENMIN values.
|
||||
(But don't forget the preprocessing steps by substrings.pl before
|
||||
concatenation.) One of the disadvantage of this method, that PATGEN
|
||||
doesn't know recursive compound hyphenation of Hyphen.
|
||||
|
||||
László Németh
|
||||
<nemeth (at) openoffice.org>
|
||||
|
|
@ -1,108 +0,0 @@
|
|||
Brief explanation of the hyphenation algorithm herein.[1]
|
||||
|
||||
Raph Levien <raph@acm.org>
|
||||
4 Aug 1998
|
||||
|
||||
The hyphenation algorithm is basically the same as Knuth's TeX
|
||||
algorithm. However, the implementation is quite a bit faster.
|
||||
|
||||
The hyphenation files from TeX can almost be used directly. There
|
||||
is a preprocessing step, however. If you don't do the preprocessing
|
||||
step, you'll get bad hyphenations (i.e. a silent failure).
|
||||
|
||||
Start with a file such as hyphen.us. This is the TeX ushyph1.tex
|
||||
file, with the exception dictionary encoded using the same rules as
|
||||
the main portion of the file. Any line beginning with % is a comment.
|
||||
Each other line should contain exactly one rule.
|
||||
|
||||
Then, do the preprocessing - "perl substrings.pl hyphen.us". The
|
||||
resulting file is hyphen.mashed. It's in Perl, and it's fairly slow
|
||||
(it uses brute force algorithms; about 17 seconds on a P100), but it
|
||||
could probably be redone in C with clever algorithms. This would be
|
||||
valuable, for example, if it was handle user-supplied exception
|
||||
dictionaries by integrating them into the rule table.[2]
|
||||
|
||||
Once the rules are preprocessed, loading them is quite quick -
|
||||
about 200ms on a P100. It then hyphenates at about 40,000 words per
|
||||
second on a P100. I haven't benchmarked it against other
|
||||
implementations (both TeX and groff contain essentially the same
|
||||
algorithm), but expect that it runs quite a bit faster than any of
|
||||
them.
|
||||
|
||||
Knuth's algorithm
|
||||
|
||||
This section contains a brief explanation of Knuth's algorithm, in
|
||||
case you missed it from the TeX books. We'll use the semi-word
|
||||
"example" as our running example.
|
||||
|
||||
Since the beginning and end of a word are special, the algorithm is
|
||||
actually run over the prepared word (prep_word in the source)
|
||||
".example.". Knuths algorithm basically just does pattern matches from
|
||||
the rule set, then applies the matches. The patterns in this case that
|
||||
match are "xa", "xam", "mp", and "pl". These are actually stored as
|
||||
"x1a", "xam3", "4m1p", and "1p2l2". Whenever numbers appear between
|
||||
the letters, they are added in. If two (or more) patterns have numbers
|
||||
in the same place, the highest number wins. Here's the example:
|
||||
|
||||
. e x a m p l e .
|
||||
x1a
|
||||
x a m3
|
||||
4m1p
|
||||
1p2l2
|
||||
-----------------
|
||||
. e x1a4m3p2l2e .
|
||||
|
||||
Finally, hyphens are placed wherever odd numbers appear. They are,
|
||||
however, suppressed after the first letter and before the last letter
|
||||
of the word (TeX actually suppresses them before the next-to-last, as
|
||||
well). So, it's "ex-am-ple", which is correct.
|
||||
|
||||
Knuth uses a trie to implement this. I.e. he stores each rule in a
|
||||
trie structure. For each position in the word, he searches the trie,
|
||||
searching for a match. Most patterns are short, so efficiency should
|
||||
be quite good.
|
||||
|
||||
Theory of the algorithm
|
||||
|
||||
The algorithm works as a slightly modified finite state machine.
|
||||
There are two kinds of transitions: those that consume one letter of
|
||||
input (which work just like your regular finite state machine), and
|
||||
"fallback" transitions, which don't consume any input. If no
|
||||
transition matching the next letter is found, the fallback is used.
|
||||
One way of looking at this is a form of compression of the transition
|
||||
tables - i.e. it behaves the same as a completely vanilla state
|
||||
machine in which the actual transition table of a node is made up of
|
||||
the union of transition tables of the node itself, plus its fallbacks.
|
||||
|
||||
Each state is represented by a string. Thus, if the current state
|
||||
is "am" and the next letter is "p", then the next state is "amp".
|
||||
Fallback transitions go to states which chop off one or (sometimes)
|
||||
more letters from the beginning. For example, if none of the
|
||||
transitions from "amp" match the next letter, then it will fall back
|
||||
to "mp". Similarly, if none of the transitions from "mp" match the
|
||||
next letter, it will fall back to "m".
|
||||
|
||||
Each state is also associated with a (possibly null) "match"
|
||||
string. This represents the union of all patterns which are
|
||||
right-justified substrings of the match string. I.e. the pattern "mp"
|
||||
is a right-justified substring of the state "amp", so it's numbers get
|
||||
added in. The actual calculation of this union is done by the
|
||||
Perl preprocessing script, but could probably be done in C just about
|
||||
as easily.
|
||||
|
||||
Because each state transition either consumes one input character
|
||||
or shortens the state string by one character, the total number of
|
||||
state transitions is linear in the length of the word.
|
||||
|
||||
[1] Documentations:
|
||||
|
||||
Franklin M. Liang: Word Hy-phen-a-tion by Com-put-er.
|
||||
Stanford University, 1983. http://www.tug.org/docs/liang.
|
||||
|
||||
László Németh: Automatic non-standard hyphenation in OpenOffice.org,
|
||||
TUGboat (27), 2006. No. 2., http://hunspell.sourceforge.net/tb87nemeth.pdf
|
||||
|
||||
[2] There is the C version of pattern converter "substrings.c"
|
||||
in the distribution written by Nanning Buitenhuis. Unfortunatelly,
|
||||
this version hasn't handled the non standard extension of the
|
||||
algorithm, yet.
|
||||
|
|
@ -1,122 +0,0 @@
|
|||
Non-standard hyphenation
|
||||
------------------------
|
||||
|
||||
Some languages use non-standard hyphenation; `discretionary'
|
||||
character changes at hyphenation points. For example,
|
||||
Catalan: paral·lel -> paral-lel,
|
||||
Dutch: omaatje -> oma-tje,
|
||||
German (before the new orthography): Schiffahrt -> Schiff-fahrt,
|
||||
Hungarian: asszonnyal -> asz-szony-nyal (multiple occurance!)
|
||||
Swedish: tillata -> till-lata.
|
||||
|
||||
Using this extended library, you can define
|
||||
non-standard hyphenation patterns. For example:
|
||||
|
||||
l·1l/l=l
|
||||
a1atje./a=t,1,3
|
||||
.schif1fahrt/ff=f,5,2
|
||||
.as3szon/sz=sz,2,3
|
||||
n1nyal./ny=ny,1,3
|
||||
.til1lata./ll=l,3,2
|
||||
|
||||
or with narrow boundaries:
|
||||
|
||||
l·1l/l=,1,2
|
||||
a1atje./a=,1,1
|
||||
.schif1fahrt/ff=,5,1
|
||||
.as3szon/sz=,2,1
|
||||
n1nyal./ny=,1,1
|
||||
.til1lata./ll=,3,1
|
||||
|
||||
Note: Libhnj uses modified patterns by preparing substrings.pl.
|
||||
Unfortunatelly, now the conversion step can generate bad non-standard
|
||||
patterns (non-standard -> standard pattern conversion), so using
|
||||
narrow boundaries may be better for recent Libhnj. For example,
|
||||
substrings.pl generates a few bad patterns for Hungarian hyphenation
|
||||
patterns resulting bad non-standard hyphenation in a few cases. Using narrow
|
||||
boundaries solves this problem. Java HyFo module can check this problem.
|
||||
|
||||
Syntax of the non-standard hyphenation patterns
|
||||
------------------------------------------------
|
||||
|
||||
pat1tern/change[,start,cut]
|
||||
|
||||
If this pattern matches the word, and this pattern win (see README.hyphen)
|
||||
in the change region of the pattern, then pattern[start, start + cut - 1]
|
||||
substring will be replaced with the "change".
|
||||
|
||||
For example, a German ff -> ff-f hyphenation:
|
||||
|
||||
f1f/ff=f
|
||||
|
||||
or with expansion
|
||||
|
||||
f1f/ff=f,1,2
|
||||
|
||||
will change every "ff" with "ff=f" at hyphenation.
|
||||
|
||||
A more real example:
|
||||
|
||||
% simple ff -> f-f hyphenation
|
||||
f1f
|
||||
% Schiffahrt -> Schiff-fahrt hyphenation
|
||||
%
|
||||
schif3fahrt/ff=f,5,2
|
||||
|
||||
Specification
|
||||
|
||||
- Pattern: matching patterns of the original Liang's algorithm
|
||||
- patterns must contain only one hyphenation point at change region
|
||||
signed with an one-digit odd number (1, 3, 5, 7 or 9).
|
||||
These point may be at subregion boundaries: schif3fahrt/ff=,5,1
|
||||
- only the greater value guarantees the win (don't mix non-standard and
|
||||
non-standard patterns with the same value, for example
|
||||
instead of f3f and schif3fahrt/ff=f,5,2 use f3f and schif5fahrt/ff=f,5,2)
|
||||
|
||||
- Change: new characters.
|
||||
Arbitrary character sequence. Equal sign (=) signs hyphenation points
|
||||
for OpenOffice.org (like in the example). (In a possible German LaTeX
|
||||
preprocessor, ff could be replaced with "ff, for a Hungarian one, ssz
|
||||
with `ssz, according to the German and Hungarian Babel settings.)
|
||||
|
||||
- Start: starting position of the change region.
|
||||
- begins with 1 (not 0): schif3fahrt/ff=f,5,2
|
||||
- start dot doesn't matter: .schif3fahrt/ff=f,5,2
|
||||
- numbers don't matter: .s2c2h2i2f3f2ahrt/ff=f,5,2
|
||||
- In UTF-8 encoding, use Unicode character positions: össze/sz=sz,2,3
|
||||
("össze" looks "össze" in an ISO 8859-1 8-bit editor).
|
||||
|
||||
- Cut: length of the removed character sequence in the original word.
|
||||
- In UTF-8 encoding, use Unicode character length: paral·1lel/l=l,5,3
|
||||
("paral·lel" looks "paral·1lel" in an ISO 8859-1 8-bit editor).
|
||||
|
||||
Dictionary developing
|
||||
---------------------
|
||||
|
||||
There hasn't been extended PatGen pattern generator for non-standard
|
||||
hyphenation patterns, yet.
|
||||
|
||||
Fortunatelly, non-standard hyphenation points are forbidden in the PatGen
|
||||
generated hyphenation patterns, so with a little patch can be develop
|
||||
non-standard hyphenation patterns also in this case.
|
||||
|
||||
Warning: If you use UTF-8 Unicode encoding in your patterns, call
|
||||
substrings.pl with UTF-8 parameter to calculate right
|
||||
character positions for non-standard hyphenation:
|
||||
|
||||
./substrings.pl input output UTF-8
|
||||
|
||||
Programming
|
||||
-----------
|
||||
|
||||
Use hyphenate2() or hyphenate3() to handle non-standard hyphenation.
|
||||
See hyphen.h for the documentation of the hyphenate*() functions.
|
||||
See example.c for processing the output of the hyphenate*() functions.
|
||||
|
||||
Warning: change characters are lower cased in the source, so you may need
|
||||
case conversion of the change characters based on input word case detection.
|
||||
For example, see OpenOffice.org source
|
||||
(lingucomponent/source/hyphenator/altlinuxhyph/hyphen/hyphenimp.cxx).
|
||||
|
||||
László Németh
|
||||
<nemeth (at) openoffice.org>
|
||||
Разница между файлами не показана из-за своего большого размера
Загрузить разницу
|
|
@ -1,175 +0,0 @@
|
|||
/* Hyphen - hyphenation library using converted TeX hyphenation patterns
|
||||
*
|
||||
* (C) 1998 Raph Levien
|
||||
* (C) 2001 ALTLinux, Moscow
|
||||
* (C) 2006, 2007, 2008 László Németh
|
||||
*
|
||||
* This was part of libHnj library by Raph Levien.
|
||||
*
|
||||
* Peter Novodvorsky from ALTLinux cut hyphenation part from libHnj
|
||||
* to use it in OpenOffice.org.
|
||||
*
|
||||
* Non-standard and compound word hyphenation support by László Németh.
|
||||
*
|
||||
* License is the original LibHnj license:
|
||||
*
|
||||
* LibHnj is dual licensed under LGPL and MPL. Boilerplate for both
|
||||
* licenses follows.
|
||||
*/
|
||||
|
||||
/* LibHnj - a library for high quality hyphenation and justification
|
||||
* Copyright (C) 1998 Raph Levien
|
||||
*
|
||||
* This library is free software; you can redistribute it and/or
|
||||
* modify it under the terms of the GNU Library General Public
|
||||
* License as published by the Free Software Foundation; either
|
||||
* version 2 of the License, or (at your option) any later version.
|
||||
*
|
||||
* This library is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
* Library General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Library General Public
|
||||
* License along with this library; if not, write to the
|
||||
* Free Software Foundation, Inc., 59 Temple Place - Suite 330,
|
||||
* Boston, MA 02111-1307 USA.
|
||||
*/
|
||||
|
||||
/*
|
||||
* The contents of this file are subject to the Mozilla Public License
|
||||
* Version 1.0 (the "MPL"); you may not use this file except in
|
||||
* compliance with the MPL. You may obtain a copy of the MPL at
|
||||
* http://www.mozilla.org/MPL/
|
||||
*
|
||||
* Software distributed under the MPL is distributed on an "AS IS" basis,
|
||||
* WITHOUT WARRANTY OF ANY KIND, either express or implied. See the MPL
|
||||
* for the specific language governing rights and limitations under the
|
||||
* MPL.
|
||||
*
|
||||
*/
|
||||
#ifndef __HYPHEN_H__
|
||||
#define __HYPHEN_H__
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif /* __cplusplus */
|
||||
|
||||
#include <stdio.h>
|
||||
|
||||
typedef struct _HyphenDict HyphenDict;
|
||||
typedef struct _HyphenState HyphenState;
|
||||
typedef struct _HyphenTrans HyphenTrans;
|
||||
#define MAX_CHARS 100
|
||||
#define MAX_NAME 20
|
||||
|
||||
struct _HyphenDict {
|
||||
/* user options */
|
||||
char lhmin; /* lefthyphenmin: min. hyph. distance from the left side */
|
||||
char rhmin; /* righthyphenmin: min. hyph. distance from the right side */
|
||||
char clhmin; /* min. hyph. distance from the left compound boundary */
|
||||
char crhmin; /* min. hyph. distance from the right compound boundary */
|
||||
char * nohyphen; /* comma separated list of characters or character
|
||||
sequences with forbidden hyphenation */
|
||||
int nohyphenl; /* count of elements in nohyphen */
|
||||
/* system variables */
|
||||
int num_states;
|
||||
char cset[MAX_NAME];
|
||||
int utf8;
|
||||
HyphenState *states;
|
||||
HyphenDict *nextlevel;
|
||||
};
|
||||
|
||||
struct _HyphenState {
|
||||
char *match;
|
||||
char *repl;
|
||||
signed char replindex;
|
||||
signed char replcut;
|
||||
int fallback_state;
|
||||
int num_trans;
|
||||
HyphenTrans *trans;
|
||||
};
|
||||
|
||||
struct _HyphenTrans {
|
||||
char ch;
|
||||
int new_state;
|
||||
};
|
||||
|
||||
HyphenDict *hnj_hyphen_load (const char *fn);
|
||||
HyphenDict *hnj_hyphen_load_file (FILE *f);
|
||||
void hnj_hyphen_free (HyphenDict *dict);
|
||||
|
||||
/* obsolete, use hnj_hyphen_hyphenate2() or *hyphenate3() functions) */
|
||||
int hnj_hyphen_hyphenate (HyphenDict *dict,
|
||||
const char *word, int word_size,
|
||||
char *hyphens);
|
||||
|
||||
/*
|
||||
|
||||
int hnj_hyphen_hyphenate2(): non-standard hyphenation.
|
||||
|
||||
(It supports Catalan, Dutch, German, Hungarian, Norwegian, Swedish
|
||||
etc. orthography, see documentation.)
|
||||
|
||||
input data:
|
||||
word: input word
|
||||
word_size: byte length of the input word
|
||||
|
||||
hyphens: allocated character buffer (size = word_size + 5)
|
||||
hyphenated_word: allocated character buffer (size ~ word_size * 2) or NULL
|
||||
rep, pos, cut: pointers (point to the allocated and _zeroed_ buffers
|
||||
(size=word_size) or with NULL value) or NULL
|
||||
|
||||
output data:
|
||||
hyphens: hyphenation vector (hyphenation points signed with odd numbers)
|
||||
hyphenated_word: hyphenated input word (hyphens signed with `='),
|
||||
optional (NULL input)
|
||||
rep: NULL (only standard hyph.), or replacements (hyphenation points
|
||||
signed with `=' in replacements);
|
||||
pos: NULL, or difference of the actual position and the beginning
|
||||
positions of the change in input words;
|
||||
cut: NULL, or counts of the removed characters of the original words
|
||||
at hyphenation,
|
||||
|
||||
Note: rep, pos, cut are complementary arrays to the hyphens, indexed with the
|
||||
character positions of the input word.
|
||||
|
||||
For example:
|
||||
Schiffahrt -> Schiff=fahrt,
|
||||
pattern: f1f/ff=f,1,2
|
||||
output: rep[5]="ff=f", pos[5] = 1, cut[5] = 2
|
||||
|
||||
Note: hnj_hyphen_hyphenate2() can allocate rep, pos, cut (word_size
|
||||
length arrays):
|
||||
|
||||
char ** rep = NULL;
|
||||
int * pos = NULL;
|
||||
int * cut = NULL;
|
||||
char hyphens[MAXWORDLEN];
|
||||
hnj_hyphen_hyphenate2(dict, "example", 7, hyphens, NULL, &rep, &pos, &cut);
|
||||
|
||||
See example in the source distribution.
|
||||
|
||||
*/
|
||||
|
||||
int hnj_hyphen_hyphenate2 (HyphenDict *dict,
|
||||
const char *word, int word_size, char * hyphens,
|
||||
char *hyphenated_word, char *** rep, int ** pos, int ** cut);
|
||||
|
||||
/* like hnj_hyphen_hyphenate2, but with hyphenmin parameters */
|
||||
/* lhmin: lefthyphenmin
|
||||
* rhmin: righthyphenmin
|
||||
* clhmin: compoundlefthyphemin
|
||||
* crhmin: compoundrighthyphenmin
|
||||
* (see documentation) */
|
||||
|
||||
int hnj_hyphen_hyphenate3 (HyphenDict *dict,
|
||||
const char *word, int word_size, char * hyphens,
|
||||
char *hyphword, char *** rep, int ** pos, int ** cut,
|
||||
int lhmin, int rhmin, int clhmin, int crhmin);
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif /* __cplusplus */
|
||||
|
||||
#endif /* __HYPHEN_H__ */
|
||||
|
|
@ -1,19 +0,0 @@
|
|||
# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*-
|
||||
# vim: set filetype=python:
|
||||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
|
||||
# These files cannot be built in unified mode because they include hnjalloc.h.
|
||||
SOURCES += [
|
||||
'hyphen.c',
|
||||
]
|
||||
|
||||
FINAL_LIBRARY = 'xul'
|
||||
|
||||
LOCAL_INCLUDES += [
|
||||
'../glue',
|
||||
]
|
||||
|
||||
# We allow warnings for third-party code that can be updated from upstream.
|
||||
AllowCompilerWarnings()
|
||||
Двоичный файл не отображается.
Двоичный файл не отображается.
Двоичный файл не отображается.
Двоичный файл не отображается.
Двоичный файл не отображается.
Двоичный файл не отображается.
Двоичный файл не отображается.
Двоичный файл не отображается.
Двоичный файл не отображается.
Двоичный файл не отображается.
Двоичный файл не отображается.
Двоичный файл не отображается.
Двоичный файл не отображается.
Двоичный файл не отображается.
Двоичный файл не отображается.
Двоичный файл не отображается.
|
|
@ -1589,8 +1589,8 @@ izn4j
|
|||
iz1no
|
||||
2z1p
|
||||
2z1s
|
||||
.W8a8r9s8z8a9w8a.
|
||||
.Warsza3w2a
|
||||
.w8a8r9s8z8a9w8a.
|
||||
.warsza3w2a
|
||||
.d8o9z8n8a.
|
||||
.do1z2na1
|
||||
.n8j8e8j9s8y8m.
|
||||
|
|
|
|||
Двоичный файл не отображается.
Двоичный файл не отображается.
Двоичный файл не отображается.
Двоичный файл не отображается.
Двоичный файл не отображается.
Двоичный файл не отображается.
Двоичный файл не отображается.
Двоичный файл не отображается.
Двоичный файл не отображается.
|
|
@ -42,7 +42,7 @@ locales = [
|
|||
'tr',
|
||||
'uk',
|
||||
]
|
||||
filename = '{locale}/hyphenation/hyph_{locale}.dic'
|
||||
filename = '{locale}/hyphenation/hyph_{locale}.hyf'
|
||||
FINAL_TARGET_FILES.hyphenation += [filename.format(locale=locale) for locale in locales]
|
||||
# en-US is a special case: the dic file is named like en_US.
|
||||
FINAL_TARGET_FILES.hyphenation += ['en-US/hyphenation/hyph_en_US.dic']
|
||||
FINAL_TARGET_FILES.hyphenation += ['en-US/hyphenation/hyph_en_US.hyf']
|
||||
|
|
|
|||
Двоичный файл не отображается.
Двоичный файл не отображается.
Двоичный файл не отображается.
Двоичный файл не отображается.
Двоичный файл не отображается.
Двоичный файл не отображается.
Двоичный файл не отображается.
Двоичный файл не отображается.
Двоичный файл не отображается.
Двоичный файл не отображается.
Двоичный файл не отображается.
|
|
@ -9,7 +9,6 @@ TEST_DIRS += [
|
|||
]
|
||||
|
||||
DIRS += [
|
||||
'hyphenation/hyphen',
|
||||
'hyphenation/glue',
|
||||
'locale',
|
||||
'locales',
|
||||
|
|
|
|||
|
|
@ -29,7 +29,8 @@ def generate(output, cbindgen_crate_path, *in_tree_dependencies):
|
|||
"--lockfile",
|
||||
CARGO_LOCK,
|
||||
"--crate",
|
||||
_get_crate_name(cbindgen_crate_path)
|
||||
_get_crate_name(cbindgen_crate_path),
|
||||
"--cpp-compat"
|
||||
], env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
|
||||
|
||||
stdout, stderr = p.communicate()
|
||||
|
|
|
|||
|
|
@ -0,0 +1 @@
|
|||
{"files":{"Cargo.toml":"029c7ee250eebc5214429f52191b2727e93abb48af335bedb8990b3a295abd13","LICENSE":"fab3dd6bdab226f1c08630b1dd917e11fcb4ec5e1e020e2c16f83a0a13863e85","README.md":"e4470dde98b9a446f357f81744b6a75b881c6490d91140df904cad0a4c4a9678","benches/bench.rs":"81699baf3431a4c50886c2f19c12873cdefa66af39992f73ea77e1e4df5c4035","cbindgen.toml":"9fc747f61df8118ad282c4951ef330aa48b5bbf53b5e36c22aec1360324762c1","doc/mapped_hyph_format.md":"2f2487cf536fe4b03db6e4b384be06744ec30b3f299519492288306a93127fbb","hyph_en_US.hyf":"6262b4c5118fe277ab4add8689d9524ca72097564652baec67a8fcd5029ec9b0","src/bin/hyf_compile.rs":"7c14426d1e7bf608aa16c120a582e3818a6977dce103cc01cd52201f9ae7f3d5","src/builder.rs":"2f7018a33fce3b096f2667829b341ace5c4681dbdd89ce9e7454790d7ad3a543","src/ffi.rs":"87b09259a98ec3f394846450162ed672192dc07fefcd90fd1c1432d26ac23865","src/lib.rs":"1d6513218858c2d89bf753a3e2bede275842aef33796ac1a5a1b650d4f4cdb2e","src/main.rs":"742d1fb78959cea5bfb6222fd420b5dde6b936671a3cc19e5313a6a53ecf69da","tests/base.hyf":"d8bf57c6280cfa1d357d3fdba156ce64afbd9df58e28eeb084dfe3f80972b73f","tests/base.hyph":"a3f1fab24c101701fdf21e8359685d80611ab970304e2bd89ef024768b3700c8","tests/base.word":"1136c9a421b242262661b9a65723f87a5ecf77ae38eabcea057832d036d567fd","tests/compound.hyf":"929c1ba6676e4c43bc649d0abf4275ea9e8b02bffaa5acdf704a710813a7a13c","tests/compound4.hyf":"2093287bc41ee30ff9bdbf278f1f8209cb1d1a78236b46e9060af2a881572b8e","tests/compound5.hyf":"0942a5dfbb8d0ef3a937ab9da0418abb41300357cde49f4c477a59a11b2cb6bd","tests/compound6.hyf":"ebad958c2692a5b439b31e324020ed27c42dc05bd5b8c6a6dea4669e6ccf76b4","tests/hyphen.hyf":"92b8a5c86aac6a0b9f0eb7330a057065d6985fd047e851cae47039995c682d4d","tests/lhmin.hyf":"23c886704fafee7d9c54b2478029cf69a5fa946c2f2442bd86697bca5933c88d","tests/num.hyf":"4834fabe78b5c81815434d4562ce3322541649e1ea1edc555a498574bc8b237e","tests/rhmin.hyf":"239cb3d4d7f904abb43b57241e12cc1396e636220c3806e64666aca7ca46cc42","tests/settings2.hyf":"9fc4855e0b952a3593db1efef080b93ce7f1c6fe6798db0440e2bf0cc986ffa2","tests/settings3.hyf":"867db207b485a06e7d60ad10735c9111f10516ee3a5afd6306c683ace3454491","tests/test.rs":"48e472835d8795def704a140aba07d01c2e5bd59ccfe683214493c5514b939f8"},"package":null}
|
||||
|
|
@ -0,0 +1,13 @@
|
|||
[package]
|
||||
name = "mapped_hyph"
|
||||
description = "Hyphenation using precompiled memory-mapped tables"
|
||||
version = "0.2.0"
|
||||
authors = ["Jonathan Kew <jfkthame@gmail.com>"]
|
||||
license = "MPL-2.0"
|
||||
|
||||
[dependencies]
|
||||
memmap = "0.7.0"
|
||||
arrayref = "0.3.5"
|
||||
|
||||
[dev-dependencies]
|
||||
lazy_static = "1.2.0"
|
||||
|
|
@ -0,0 +1,373 @@
|
|||
Mozilla Public License Version 2.0
|
||||
==================================
|
||||
|
||||
1. Definitions
|
||||
--------------
|
||||
|
||||
1.1. "Contributor"
|
||||
means each individual or legal entity that creates, contributes to
|
||||
the creation of, or owns Covered Software.
|
||||
|
||||
1.2. "Contributor Version"
|
||||
means the combination of the Contributions of others (if any) used
|
||||
by a Contributor and that particular Contributor's Contribution.
|
||||
|
||||
1.3. "Contribution"
|
||||
means Covered Software of a particular Contributor.
|
||||
|
||||
1.4. "Covered Software"
|
||||
means Source Code Form to which the initial Contributor has attached
|
||||
the notice in Exhibit A, the Executable Form of such Source Code
|
||||
Form, and Modifications of such Source Code Form, in each case
|
||||
including portions thereof.
|
||||
|
||||
1.5. "Incompatible With Secondary Licenses"
|
||||
means
|
||||
|
||||
(a) that the initial Contributor has attached the notice described
|
||||
in Exhibit B to the Covered Software; or
|
||||
|
||||
(b) that the Covered Software was made available under the terms of
|
||||
version 1.1 or earlier of the License, but not also under the
|
||||
terms of a Secondary License.
|
||||
|
||||
1.6. "Executable Form"
|
||||
means any form of the work other than Source Code Form.
|
||||
|
||||
1.7. "Larger Work"
|
||||
means a work that combines Covered Software with other material, in
|
||||
a separate file or files, that is not Covered Software.
|
||||
|
||||
1.8. "License"
|
||||
means this document.
|
||||
|
||||
1.9. "Licensable"
|
||||
means having the right to grant, to the maximum extent possible,
|
||||
whether at the time of the initial grant or subsequently, any and
|
||||
all of the rights conveyed by this License.
|
||||
|
||||
1.10. "Modifications"
|
||||
means any of the following:
|
||||
|
||||
(a) any file in Source Code Form that results from an addition to,
|
||||
deletion from, or modification of the contents of Covered
|
||||
Software; or
|
||||
|
||||
(b) any new file in Source Code Form that contains any Covered
|
||||
Software.
|
||||
|
||||
1.11. "Patent Claims" of a Contributor
|
||||
means any patent claim(s), including without limitation, method,
|
||||
process, and apparatus claims, in any patent Licensable by such
|
||||
Contributor that would be infringed, but for the grant of the
|
||||
License, by the making, using, selling, offering for sale, having
|
||||
made, import, or transfer of either its Contributions or its
|
||||
Contributor Version.
|
||||
|
||||
1.12. "Secondary License"
|
||||
means either the GNU General Public License, Version 2.0, the GNU
|
||||
Lesser General Public License, Version 2.1, the GNU Affero General
|
||||
Public License, Version 3.0, or any later versions of those
|
||||
licenses.
|
||||
|
||||
1.13. "Source Code Form"
|
||||
means the form of the work preferred for making modifications.
|
||||
|
||||
1.14. "You" (or "Your")
|
||||
means an individual or a legal entity exercising rights under this
|
||||
License. For legal entities, "You" includes any entity that
|
||||
controls, is controlled by, or is under common control with You. For
|
||||
purposes of this definition, "control" means (a) the power, direct
|
||||
or indirect, to cause the direction or management of such entity,
|
||||
whether by contract or otherwise, or (b) ownership of more than
|
||||
fifty percent (50%) of the outstanding shares or beneficial
|
||||
ownership of such entity.
|
||||
|
||||
2. License Grants and Conditions
|
||||
--------------------------------
|
||||
|
||||
2.1. Grants
|
||||
|
||||
Each Contributor hereby grants You a world-wide, royalty-free,
|
||||
non-exclusive license:
|
||||
|
||||
(a) under intellectual property rights (other than patent or trademark)
|
||||
Licensable by such Contributor to use, reproduce, make available,
|
||||
modify, display, perform, distribute, and otherwise exploit its
|
||||
Contributions, either on an unmodified basis, with Modifications, or
|
||||
as part of a Larger Work; and
|
||||
|
||||
(b) under Patent Claims of such Contributor to make, use, sell, offer
|
||||
for sale, have made, import, and otherwise transfer either its
|
||||
Contributions or its Contributor Version.
|
||||
|
||||
2.2. Effective Date
|
||||
|
||||
The licenses granted in Section 2.1 with respect to any Contribution
|
||||
become effective for each Contribution on the date the Contributor first
|
||||
distributes such Contribution.
|
||||
|
||||
2.3. Limitations on Grant Scope
|
||||
|
||||
The licenses granted in this Section 2 are the only rights granted under
|
||||
this License. No additional rights or licenses will be implied from the
|
||||
distribution or licensing of Covered Software under this License.
|
||||
Notwithstanding Section 2.1(b) above, no patent license is granted by a
|
||||
Contributor:
|
||||
|
||||
(a) for any code that a Contributor has removed from Covered Software;
|
||||
or
|
||||
|
||||
(b) for infringements caused by: (i) Your and any other third party's
|
||||
modifications of Covered Software, or (ii) the combination of its
|
||||
Contributions with other software (except as part of its Contributor
|
||||
Version); or
|
||||
|
||||
(c) under Patent Claims infringed by Covered Software in the absence of
|
||||
its Contributions.
|
||||
|
||||
This License does not grant any rights in the trademarks, service marks,
|
||||
or logos of any Contributor (except as may be necessary to comply with
|
||||
the notice requirements in Section 3.4).
|
||||
|
||||
2.4. Subsequent Licenses
|
||||
|
||||
No Contributor makes additional grants as a result of Your choice to
|
||||
distribute the Covered Software under a subsequent version of this
|
||||
License (see Section 10.2) or under the terms of a Secondary License (if
|
||||
permitted under the terms of Section 3.3).
|
||||
|
||||
2.5. Representation
|
||||
|
||||
Each Contributor represents that the Contributor believes its
|
||||
Contributions are its original creation(s) or it has sufficient rights
|
||||
to grant the rights to its Contributions conveyed by this License.
|
||||
|
||||
2.6. Fair Use
|
||||
|
||||
This License is not intended to limit any rights You have under
|
||||
applicable copyright doctrines of fair use, fair dealing, or other
|
||||
equivalents.
|
||||
|
||||
2.7. Conditions
|
||||
|
||||
Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted
|
||||
in Section 2.1.
|
||||
|
||||
3. Responsibilities
|
||||
-------------------
|
||||
|
||||
3.1. Distribution of Source Form
|
||||
|
||||
All distribution of Covered Software in Source Code Form, including any
|
||||
Modifications that You create or to which You contribute, must be under
|
||||
the terms of this License. You must inform recipients that the Source
|
||||
Code Form of the Covered Software is governed by the terms of this
|
||||
License, and how they can obtain a copy of this License. You may not
|
||||
attempt to alter or restrict the recipients' rights in the Source Code
|
||||
Form.
|
||||
|
||||
3.2. Distribution of Executable Form
|
||||
|
||||
If You distribute Covered Software in Executable Form then:
|
||||
|
||||
(a) such Covered Software must also be made available in Source Code
|
||||
Form, as described in Section 3.1, and You must inform recipients of
|
||||
the Executable Form how they can obtain a copy of such Source Code
|
||||
Form by reasonable means in a timely manner, at a charge no more
|
||||
than the cost of distribution to the recipient; and
|
||||
|
||||
(b) You may distribute such Executable Form under the terms of this
|
||||
License, or sublicense it under different terms, provided that the
|
||||
license for the Executable Form does not attempt to limit or alter
|
||||
the recipients' rights in the Source Code Form under this License.
|
||||
|
||||
3.3. Distribution of a Larger Work
|
||||
|
||||
You may create and distribute a Larger Work under terms of Your choice,
|
||||
provided that You also comply with the requirements of this License for
|
||||
the Covered Software. If the Larger Work is a combination of Covered
|
||||
Software with a work governed by one or more Secondary Licenses, and the
|
||||
Covered Software is not Incompatible With Secondary Licenses, this
|
||||
License permits You to additionally distribute such Covered Software
|
||||
under the terms of such Secondary License(s), so that the recipient of
|
||||
the Larger Work may, at their option, further distribute the Covered
|
||||
Software under the terms of either this License or such Secondary
|
||||
License(s).
|
||||
|
||||
3.4. Notices
|
||||
|
||||
You may not remove or alter the substance of any license notices
|
||||
(including copyright notices, patent notices, disclaimers of warranty,
|
||||
or limitations of liability) contained within the Source Code Form of
|
||||
the Covered Software, except that You may alter any license notices to
|
||||
the extent required to remedy known factual inaccuracies.
|
||||
|
||||
3.5. Application of Additional Terms
|
||||
|
||||
You may choose to offer, and to charge a fee for, warranty, support,
|
||||
indemnity or liability obligations to one or more recipients of Covered
|
||||
Software. However, You may do so only on Your own behalf, and not on
|
||||
behalf of any Contributor. You must make it absolutely clear that any
|
||||
such warranty, support, indemnity, or liability obligation is offered by
|
||||
You alone, and You hereby agree to indemnify every Contributor for any
|
||||
liability incurred by such Contributor as a result of warranty, support,
|
||||
indemnity or liability terms You offer. You may include additional
|
||||
disclaimers of warranty and limitations of liability specific to any
|
||||
jurisdiction.
|
||||
|
||||
4. Inability to Comply Due to Statute or Regulation
|
||||
---------------------------------------------------
|
||||
|
||||
If it is impossible for You to comply with any of the terms of this
|
||||
License with respect to some or all of the Covered Software due to
|
||||
statute, judicial order, or regulation then You must: (a) comply with
|
||||
the terms of this License to the maximum extent possible; and (b)
|
||||
describe the limitations and the code they affect. Such description must
|
||||
be placed in a text file included with all distributions of the Covered
|
||||
Software under this License. Except to the extent prohibited by statute
|
||||
or regulation, such description must be sufficiently detailed for a
|
||||
recipient of ordinary skill to be able to understand it.
|
||||
|
||||
5. Termination
|
||||
--------------
|
||||
|
||||
5.1. The rights granted under this License will terminate automatically
|
||||
if You fail to comply with any of its terms. However, if You become
|
||||
compliant, then the rights granted under this License from a particular
|
||||
Contributor are reinstated (a) provisionally, unless and until such
|
||||
Contributor explicitly and finally terminates Your grants, and (b) on an
|
||||
ongoing basis, if such Contributor fails to notify You of the
|
||||
non-compliance by some reasonable means prior to 60 days after You have
|
||||
come back into compliance. Moreover, Your grants from a particular
|
||||
Contributor are reinstated on an ongoing basis if such Contributor
|
||||
notifies You of the non-compliance by some reasonable means, this is the
|
||||
first time You have received notice of non-compliance with this License
|
||||
from such Contributor, and You become compliant prior to 30 days after
|
||||
Your receipt of the notice.
|
||||
|
||||
5.2. If You initiate litigation against any entity by asserting a patent
|
||||
infringement claim (excluding declaratory judgment actions,
|
||||
counter-claims, and cross-claims) alleging that a Contributor Version
|
||||
directly or indirectly infringes any patent, then the rights granted to
|
||||
You by any and all Contributors for the Covered Software under Section
|
||||
2.1 of this License shall terminate.
|
||||
|
||||
5.3. In the event of termination under Sections 5.1 or 5.2 above, all
|
||||
end user license agreements (excluding distributors and resellers) which
|
||||
have been validly granted by You or Your distributors under this License
|
||||
prior to termination shall survive termination.
|
||||
|
||||
************************************************************************
|
||||
* *
|
||||
* 6. Disclaimer of Warranty *
|
||||
* ------------------------- *
|
||||
* *
|
||||
* Covered Software is provided under this License on an "as is" *
|
||||
* basis, without warranty of any kind, either expressed, implied, or *
|
||||
* statutory, including, without limitation, warranties that the *
|
||||
* Covered Software is free of defects, merchantable, fit for a *
|
||||
* particular purpose or non-infringing. The entire risk as to the *
|
||||
* quality and performance of the Covered Software is with You. *
|
||||
* Should any Covered Software prove defective in any respect, You *
|
||||
* (not any Contributor) assume the cost of any necessary servicing, *
|
||||
* repair, or correction. This disclaimer of warranty constitutes an *
|
||||
* essential part of this License. No use of any Covered Software is *
|
||||
* authorized under this License except under this disclaimer. *
|
||||
* *
|
||||
************************************************************************
|
||||
|
||||
************************************************************************
|
||||
* *
|
||||
* 7. Limitation of Liability *
|
||||
* -------------------------- *
|
||||
* *
|
||||
* Under no circumstances and under no legal theory, whether tort *
|
||||
* (including negligence), contract, or otherwise, shall any *
|
||||
* Contributor, or anyone who distributes Covered Software as *
|
||||
* permitted above, be liable to You for any direct, indirect, *
|
||||
* special, incidental, or consequential damages of any character *
|
||||
* including, without limitation, damages for lost profits, loss of *
|
||||
* goodwill, work stoppage, computer failure or malfunction, or any *
|
||||
* and all other commercial damages or losses, even if such party *
|
||||
* shall have been informed of the possibility of such damages. This *
|
||||
* limitation of liability shall not apply to liability for death or *
|
||||
* personal injury resulting from such party's negligence to the *
|
||||
* extent applicable law prohibits such limitation. Some *
|
||||
* jurisdictions do not allow the exclusion or limitation of *
|
||||
* incidental or consequential damages, so this exclusion and *
|
||||
* limitation may not apply to You. *
|
||||
* *
|
||||
************************************************************************
|
||||
|
||||
8. Litigation
|
||||
-------------
|
||||
|
||||
Any litigation relating to this License may be brought only in the
|
||||
courts of a jurisdiction where the defendant maintains its principal
|
||||
place of business and such litigation shall be governed by laws of that
|
||||
jurisdiction, without reference to its conflict-of-law provisions.
|
||||
Nothing in this Section shall prevent a party's ability to bring
|
||||
cross-claims or counter-claims.
|
||||
|
||||
9. Miscellaneous
|
||||
----------------
|
||||
|
||||
This License represents the complete agreement concerning the subject
|
||||
matter hereof. If any provision of this License is held to be
|
||||
unenforceable, such provision shall be reformed only to the extent
|
||||
necessary to make it enforceable. Any law or regulation which provides
|
||||
that the language of a contract shall be construed against the drafter
|
||||
shall not be used to construe this License against a Contributor.
|
||||
|
||||
10. Versions of the License
|
||||
---------------------------
|
||||
|
||||
10.1. New Versions
|
||||
|
||||
Mozilla Foundation is the license steward. Except as provided in Section
|
||||
10.3, no one other than the license steward has the right to modify or
|
||||
publish new versions of this License. Each version will be given a
|
||||
distinguishing version number.
|
||||
|
||||
10.2. Effect of New Versions
|
||||
|
||||
You may distribute the Covered Software under the terms of the version
|
||||
of the License under which You originally received the Covered Software,
|
||||
or under the terms of any subsequent version published by the license
|
||||
steward.
|
||||
|
||||
10.3. Modified Versions
|
||||
|
||||
If you create software not governed by this License, and you want to
|
||||
create a new license for such software, you may create and use a
|
||||
modified version of this License if you rename the license and remove
|
||||
any references to the name of the license steward (except to note that
|
||||
such modified license differs from this License).
|
||||
|
||||
10.4. Distributing Source Code Form that is Incompatible With Secondary
|
||||
Licenses
|
||||
|
||||
If You choose to distribute Source Code Form that is Incompatible With
|
||||
Secondary Licenses under the terms of this version of the License, the
|
||||
notice described in Exhibit B of this License must be attached.
|
||||
|
||||
Exhibit A - Source Code Form License Notice
|
||||
-------------------------------------------
|
||||
|
||||
This Source Code Form is subject to the terms of the Mozilla Public
|
||||
License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
|
||||
If it is not possible or desirable to put the notice in a particular
|
||||
file, then You may include the notice in a location (such as a LICENSE
|
||||
file in a relevant directory) where a recipient would be likely to look
|
||||
for such a notice.
|
||||
|
||||
You may add additional accurate notices of copyright ownership.
|
||||
|
||||
Exhibit B - "Incompatible With Secondary Licenses" Notice
|
||||
---------------------------------------------------------
|
||||
|
||||
This Source Code Form is "Incompatible With Secondary Licenses", as
|
||||
defined by the Mozilla Public License, v. 2.0.
|
||||
|
|
@ -0,0 +1,76 @@
|
|||
# mapped_hyph
|
||||
|
||||
mapped_hyph is a reimplementation of the hyphenation algorithm from the
|
||||
[libhyphen](https://github.com/hunspell/hyphen) library
|
||||
that is intended to reduce the in-memory footprint of loaded
|
||||
hyphenation dictionaries, especially when the same dictionary
|
||||
may be in use by multiple processes.
|
||||
|
||||
To reduce memory footprint, mapped_hyph uses hyphenation dictionaries that are
|
||||
"precompiled" into a flat, position-independent binary format that is used
|
||||
directly by the runtime hyphenation functions.
|
||||
Therefore, dictionaries do not have to be parsed into a dynamic structure in memory;
|
||||
the files can simply be mmap'd into the address space and immediately used.
|
||||
In addition, a compiled dictionary mapped into a shared-memory block
|
||||
can be made available to multiple processes for no added physical memory cost.
|
||||
|
||||
One deliberate simplification compared to libhyphen
|
||||
is that mapped_hyph only accepts UTF-8 text and hyphenation dictionaries;
|
||||
legacy non-Unicode encodings are not supported.
|
||||
|
||||
mapped_hyph has been created primarily for use by Gecko, replacing the use of libhyphen,
|
||||
and so its features (and limitations) are based on this use case.
|
||||
However, it is hoped that it will also be more generally useful.
|
||||
|
||||
## Functionality
|
||||
|
||||
Currently, mapped_hyph supports only "standard" hyphenation, where spelling does not
|
||||
change around the hyphenation position. At present this is the only kind of
|
||||
hyphenation supported in Gecko.
|
||||
|
||||
The compiled hyphenation dictionary format includes provision for replacement
|
||||
strings and indexes, as used by libhyphen to support non-standard hyphenations
|
||||
(e.g. German "Schiffahrt" -> "Schiff-fahrt"), but the `find_hyphen_values` function
|
||||
will ignore any such hyphenation positions it finds.
|
||||
(None of the hyphenation dictionaries shipping with Firefox includes such rules.)
|
||||
|
||||
## Licensing
|
||||
|
||||
Please see the file named
|
||||
[LICENSE](https://github.com/jfkthame/mapped_hyph/blob/master/LICENSE),
|
||||
or the [MPL 2.0](https://www.mozilla.org/en-US/MPL/2.0/) license online.
|
||||
|
||||
## Documentation
|
||||
|
||||
Use `cargo doc --open` to view (admittedly brief) documentation generated from
|
||||
comments in the source.
|
||||
|
||||
## C and C++ bindings
|
||||
|
||||
See the `mapped_hyph.h` header for C/C++ APIs that can be used to load hyphenation files
|
||||
and to locate valid hyphenation positions in a word.
|
||||
|
||||
## Sample programs
|
||||
|
||||
See main.rs for a simple example program.
|
||||
|
||||
## Compiled dictionaries
|
||||
|
||||
The `hyf_compile` tool is used to generate `.hyf` files for mapped_hyph
|
||||
from standard `.dic` (or `.pat`) files as used by libhyphen, LibreOffice, etc.
|
||||
|
||||
(A compiled version of the `hyph_en_US` dictionary from libhyphen is currently
|
||||
included here, as it is handy for testing purposes.)
|
||||
|
||||
## Release Notes
|
||||
|
||||
### 0.2.0
|
||||
|
||||
* Implemented a hyphenation table compiler in the `builder` submodule,
|
||||
and `hyf_compile` command-line tool.
|
||||
|
||||
* Moved C-callable API functions into an `ffi` submodule.
|
||||
|
||||
### 0.1.0
|
||||
|
||||
* Initial release.
|
||||
|
|
@ -0,0 +1,33 @@
|
|||
#![feature(test)]
|
||||
extern crate test;
|
||||
|
||||
#[macro_use]
|
||||
extern crate lazy_static;
|
||||
|
||||
extern crate mapped_hyph;
|
||||
use mapped_hyph::Hyphenator;
|
||||
|
||||
lazy_static! {
|
||||
static ref WORDS: Vec<String> = {
|
||||
use std::fs::File;
|
||||
use std::io::{BufRead,BufReader};
|
||||
let file = File::open("/usr/share/dict/words").unwrap();
|
||||
BufReader::new(file).lines().map(|l| l.unwrap()).collect()
|
||||
};
|
||||
}
|
||||
|
||||
#[bench]
|
||||
fn bench_words(b: &mut test::Bencher) {
|
||||
b.iter(|| {
|
||||
let dic_path = "hyph_en_US.hyf";
|
||||
let dic = match unsafe { mapped_hyph::load_file(dic_path) } {
|
||||
Some(dic) => dic,
|
||||
_ => panic!("failed to load dictionary {}", dic_path),
|
||||
};
|
||||
let hyph = Hyphenator::new(&*dic);
|
||||
let mut values: Vec<u8> = vec![0; 1000];
|
||||
for w in WORDS.iter() {
|
||||
test::black_box(hyph.find_hyphen_values(&w, &mut values));
|
||||
}
|
||||
});
|
||||
}
|
||||
|
|
@ -0,0 +1,99 @@
|
|||
# This is a template cbindgen.toml file with all of the default values.
|
||||
# Some values are commented out because their absence is the real default.
|
||||
#
|
||||
# See https://github.com/eqrion/cbindgen/blob/master/docs.md#cbindgentoml
|
||||
# for detailed documentation of every option here.
|
||||
|
||||
language = "C"
|
||||
|
||||
############## Options for Wrapping the Contents of the Header #################
|
||||
|
||||
header = "/* This Source Code Form is subject to the terms of the Mozilla Public\n * License, v. 2.0. If a copy of the MPL was not distributed with this\n * file, You can obtain one at https://mozilla.org/MPL/2.0/. */\n\n/* clang-format off */"
|
||||
trailer = "/* clang-format on */"
|
||||
include_guard = "mapped_hyph_h"
|
||||
autogen_warning = "/* Warning, this file is autogenerated by cbindgen. Don't modify this manually.\n */"
|
||||
include_version = false
|
||||
# namespace = "my_namespace"
|
||||
namespaces = []
|
||||
# using_namespaces = []
|
||||
sys_includes = ["stdbool.h","stdint.h"]
|
||||
includes = []
|
||||
no_includes = true
|
||||
|
||||
############################ Code Style Options ################################
|
||||
|
||||
braces = "SameLine"
|
||||
line_length = 100
|
||||
tab_width = 2
|
||||
documentation_style = "auto"
|
||||
|
||||
############################# Codegen Options ##################################
|
||||
|
||||
style = "both"
|
||||
|
||||
[defines]
|
||||
# "target_os = freebsd" = "DEFINE_FREEBSD"
|
||||
# "feature = serde" = "DEFINE_SERDE"
|
||||
|
||||
[export]
|
||||
include = []
|
||||
exclude = []
|
||||
# prefix = "CAPI_"
|
||||
item_types = []
|
||||
renaming_overrides_prefixing = false
|
||||
|
||||
[export.rename]
|
||||
|
||||
[export.body]
|
||||
|
||||
[fn]
|
||||
rename_args = "None"
|
||||
# must_use = "MUST_USE_FUNC"
|
||||
# prefix = "START_FUNC"
|
||||
# postfix = "END_FUNC"
|
||||
args = "auto"
|
||||
|
||||
[struct]
|
||||
rename_fields = "None"
|
||||
# must_use = "MUST_USE_STRUCT"
|
||||
derive_constructor = false
|
||||
derive_eq = false
|
||||
derive_neq = false
|
||||
derive_lt = false
|
||||
derive_lte = false
|
||||
derive_gt = false
|
||||
derive_gte = false
|
||||
|
||||
[enum]
|
||||
rename_variants = "None"
|
||||
# must_use = "MUST_USE_ENUM"
|
||||
add_sentinel = false
|
||||
prefix_with_name = false
|
||||
derive_helper_methods = false
|
||||
derive_const_casts = false
|
||||
derive_mut_casts = false
|
||||
# cast_assert_name = "ASSERT"
|
||||
derive_tagged_enum_destructor = false
|
||||
derive_tagged_enum_copy_constructor = false
|
||||
private_default_tagged_enum_constructor = false
|
||||
|
||||
[const]
|
||||
allow_static_const = true
|
||||
|
||||
[macro_expansion]
|
||||
bitflags = false
|
||||
|
||||
############## Options for How Your Rust library Should Be Parsed ##############
|
||||
|
||||
[parse]
|
||||
parse_deps = false
|
||||
# include = []
|
||||
exclude = []
|
||||
clean = false
|
||||
extra_bindings = []
|
||||
|
||||
[parse.expand]
|
||||
crates = []
|
||||
all_features = false
|
||||
default_features = true
|
||||
features = []
|
||||
|
|
@ -0,0 +1,98 @@
|
|||
# Compiled hyphenation table format for mapped_hyph
|
||||
|
||||
The file is a "flattened" representation of the list of `HyphenDict` structs
|
||||
and descendant objects used by libhyphen
|
||||
(see [hyphen.h](https://github.com/hunspell/hyphen/blob/master/hyphen.h)).
|
||||
|
||||
Note that multi-byte integer types in the file are stored in _little-endian_ byte order.
|
||||
|
||||
## Overall file header
|
||||
|
||||
The file begins with a 4-byte "signature", followed by a count of the number
|
||||
of hyphenation levels, and an array of offsets to each hyphenation level.
|
||||
A "level" is essentially equivalent to libhyphen's `HyphenDict`.
|
||||
|
||||
### Header (size: 8 bytes + 4 * numLevels)
|
||||
Type | Name | Description
|
||||
-----|------|------------
|
||||
uint8[4] | magicNumber | 4-byte file identification code: ['H', 'y', 'f', '0']
|
||||
uint32 | numLevels | number of hyphenation levels present
|
||||
uint32[numLevels] | levelOffset | offset from start of file to each Level
|
||||
|
||||
Currently, there are normally 2 hyphenation levels, as the parser/compiler will
|
||||
generate a default first level if no NEXTLEVEL keyword is present in the pattern file.
|
||||
|
||||
## Hyphenation Level
|
||||
|
||||
Each level of the hyphenation pattern begins with a Level header, followed by
|
||||
the data for its states and the strings they refer to.
|
||||
When the hyphenation machine is executed, we always begin at state offset 0
|
||||
(from the level's stateDataBase); each transition to a new state represents the
|
||||
target directly by its offset from stateDataBase.
|
||||
A state offset of 0xFFFFFF is considered invalid.
|
||||
|
||||
Strings are represented as offsets from the level's stringDataBase; each string
|
||||
is encoded as a one-byte length followed by `length` bytes of utf-8 data.
|
||||
(So the maximum string length is 255 utf-8 code units; this is far more than any actual
|
||||
hyphenation dictionary uses).
|
||||
A string offset of 0xFFFF is considered invalid and represents an absent string.
|
||||
|
||||
The minimum number of characters that must be kept together at the start/end of a word,
|
||||
or of a component of a compound (i.e. the `...Min` values) is a count of _Unicode characters_,
|
||||
not UTF-8 code units. (Note that the presentation-form ligature characters U+FB00 'ff' through U+FB06 'st'
|
||||
are counted as 2 or 3 characters for this purpose.)
|
||||
|
||||
### Level (size: 16 bytes + state data + string data, padded to a 4-byte boundary)
|
||||
Type | Name | Description
|
||||
-----|------|------------
|
||||
uint32 | stateDataBase | offset from beginning of Level to start of level's State data
|
||||
uint32 | stringDataBase | offset from beginning of Level to start of level's packed String data
|
||||
uint16 | noHyphenStringOffset | from level's stringDataBase
|
||||
uint16 | noHyphenCount | number of (NUL-separated) strings in the nohyphen string
|
||||
uint8 | leftHyphenMin | minimum number of characters kept together at start of word
|
||||
uint8 | rightHyphenMin | minimum number of characters kept together at end of word
|
||||
uint8 | compoundLeftHyphenMin | minimum number of characters kept together at start of second component of a compound
|
||||
uint8 | compoundRightHyphenMin | minimum number of characters kept together at end of first component of a compound
|
||||
|
||||
## State
|
||||
|
||||
Each state, referred to by its offset from the level's stateDataBase, consists of a header
|
||||
followed by an array of transitions for input bytes that need to be matched in this state.
|
||||
The state also records a fallback state offset, which is the transition to be taken
|
||||
if the next input byte does not match any of the transition records.
|
||||
|
||||
If a match string is present (i.e. `matchStringOffset` is not 0xFFFF), it is a string of hyphenation values
|
||||
(encoded as ASCII digits '0'..'9') to be applied at the current position in the word.
|
||||
|
||||
### StateHeader (size: 8 bytes)
|
||||
Type | Name | Description
|
||||
-----|------|------------
|
||||
uint32 | fallbackStateOffset | (from level's stateDataBase)
|
||||
uint16 | matchStringOffset | (from level's stringDataBase)
|
||||
uint8 | numTransitions | count of Transitions that follow the StateHeader and optional StateHeaderExtension
|
||||
uint8 | isExtended | if non-zero, the StateHeader is immediately followed by a StateHeaderExtension
|
||||
|
||||
If the `isExtended` flag in the state header is set, this state includes a potential spelling change
|
||||
and there is an extended form of the header present before the array of transitions.
|
||||
(Note that extended states with spelling-change rules are not yet supported by the mapped_hyph engine;
|
||||
none of the hyphenation dictionaries shipped with Firefox includes such rules.)
|
||||
|
||||
### StateHeaderExtension (size: 4 bytes)
|
||||
Type | Name | Description
|
||||
-----|------|------------
|
||||
uint16 | replacementStringOffset | (from level's stringDataBase) the replacement string
|
||||
int8 | replacementIndex | index of the byte position (relative to current position in the word) at which the spelling replacement should happen
|
||||
int8 | replacementCut | number of bytes to cut from the original word when making the replacement
|
||||
|
||||
## Transitions
|
||||
|
||||
The state's transitions are encoded as an array of Transition records, each corresponding to an input byte
|
||||
and providing the offset of the new state. The transitions for each state are sorted by ascending value of input byte
|
||||
(although in practice there are usually only a few valid transitions, and so a binary search does not seem to be
|
||||
worthwhile).
|
||||
|
||||
### Transition (size: 4 bytes)
|
||||
Type | Name | Description
|
||||
-----|------|------------
|
||||
uint24 | newStateOffset | (from level's stateDataBase)
|
||||
uint8 | inputByte | the input byte (utf-8 code unit) for this transition
|
||||
Двоичный файл не отображается.
|
|
@ -0,0 +1,20 @@
|
|||
/* This Source Code Form is subject to the terms of the Mozilla Public
|
||||
* License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
* file, You can obtain one at https://mozilla.org/MPL/2.0/. */
|
||||
|
||||
extern crate mapped_hyph;
|
||||
|
||||
use std::env;
|
||||
use std::fs::File;
|
||||
|
||||
fn main() -> std::io::Result<()> {
|
||||
let args: Vec<String> = env::args().collect();
|
||||
if args.len() == 3 {
|
||||
let in_file = File::open(&args[1])?;
|
||||
let mut out_file = File::create(&args[2])?;
|
||||
mapped_hyph::builder::write_hyf_file(&mut out_file, mapped_hyph::builder::read_dic_file(&in_file))?;
|
||||
} else {
|
||||
println!("usage: hyf_compile <pattern-file> <output-file>");
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
|
@ -0,0 +1,469 @@
|
|||
/* This Source Code Form is subject to the terms of the Mozilla Public
|
||||
* License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
* file, You can obtain one at https://mozilla.org/MPL/2.0/. */
|
||||
|
||||
/// Functions to compile human-readable patterns into a mapped_hyph
|
||||
/// flattened representation of the hyphenation state machine.
|
||||
|
||||
use std::io::{Read,BufRead,BufReader,Write};
|
||||
use std::collections::HashMap;
|
||||
use std::convert::TryInto;
|
||||
use std::hash::{Hash,Hasher};
|
||||
|
||||
// Wrap a HashMap so that we can implement the Hash trait.
|
||||
#[derive(PartialEq, Eq, Clone)]
|
||||
struct TransitionMap (HashMap<u8,i32>);
|
||||
|
||||
impl TransitionMap {
|
||||
fn new() -> TransitionMap {
|
||||
TransitionMap(HashMap::<u8,i32>::new())
|
||||
}
|
||||
}
|
||||
|
||||
impl Hash for TransitionMap {
|
||||
fn hash<H: Hasher>(&self, state: &mut H) {
|
||||
// We only look at the values here; that's likely to be enough
|
||||
// for a reasonable hash.
|
||||
let mut transitions: Vec<&i32> = self.0.values().collect();
|
||||
transitions.sort();
|
||||
for t in transitions {
|
||||
t.hash(state);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(PartialEq, Eq, Hash, Clone)]
|
||||
struct State {
|
||||
match_string: Option<Vec<u8>>,
|
||||
#[allow(dead_code)]
|
||||
repl_string: Option<Vec<u8>>,
|
||||
#[allow(dead_code)]
|
||||
repl_index: i32,
|
||||
#[allow(dead_code)]
|
||||
repl_cut: i32,
|
||||
fallback_state: i32,
|
||||
transitions: TransitionMap,
|
||||
}
|
||||
|
||||
impl State {
|
||||
fn new() -> State {
|
||||
State {
|
||||
match_string: None,
|
||||
repl_string: None,
|
||||
repl_index: -1,
|
||||
repl_cut: -1,
|
||||
fallback_state: -1,
|
||||
transitions: TransitionMap::new(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// This is only public because the read_dic_file() function returns a Vec
|
||||
/// of LevelBuilder structs, which can then be passed to write_hyf_file()
|
||||
/// to create the flattened output.
|
||||
pub struct LevelBuilder {
|
||||
states: Vec<State>,
|
||||
str_to_state: HashMap<Vec<u8>,i32>,
|
||||
encoding: Option<String>,
|
||||
nohyphen: Option<String>,
|
||||
lh_min: u8,
|
||||
rh_min: u8,
|
||||
clh_min: u8,
|
||||
crh_min: u8,
|
||||
}
|
||||
|
||||
impl LevelBuilder {
|
||||
fn new() -> LevelBuilder {
|
||||
let mut result = LevelBuilder {
|
||||
states: Vec::<State>::new(),
|
||||
str_to_state: HashMap::<Vec<u8>,i32>::new(),
|
||||
encoding: None,
|
||||
nohyphen: None,
|
||||
lh_min: 0,
|
||||
rh_min: 0,
|
||||
clh_min: 0,
|
||||
crh_min: 0,
|
||||
};
|
||||
// Initialize the builder with an empty start state.
|
||||
result.str_to_state.insert(vec![], 0);
|
||||
result.states.push(State::new());
|
||||
result
|
||||
}
|
||||
|
||||
fn find_state_number_for(&mut self, text: &Vec<u8>) -> i32 {
|
||||
let count = self.states.len() as i32;
|
||||
let index = *self.str_to_state.entry(text.clone()).or_insert(count);
|
||||
if index == count {
|
||||
self.states.push(State::new());
|
||||
}
|
||||
index
|
||||
}
|
||||
|
||||
fn add_pattern(&mut self, pattern: &str) {
|
||||
let mut bytes = pattern.as_bytes();
|
||||
let mut text = Vec::<u8>::with_capacity(bytes.len());
|
||||
let mut digits = Vec::<u8>::with_capacity(bytes.len() + 1);
|
||||
let mut repl_str = None;
|
||||
let mut repl_index = 0;
|
||||
let mut repl_cut = 0;
|
||||
|
||||
// Check for replacement rule (non-standard hyphenation spelling change).
|
||||
if let Some(slash) = bytes.iter().position(|x| *x == b'/') {
|
||||
let parts = bytes.split_at(slash);
|
||||
bytes = parts.0;
|
||||
let mut it = parts.1[1 ..].split(|x| *x == b',');
|
||||
if let Some(repl) = it.next() {
|
||||
repl_str = Some(repl.to_vec());
|
||||
}
|
||||
if let Some(num) = it.next() {
|
||||
repl_index = std::str::from_utf8(num).unwrap().parse::<i32>().unwrap() - 1;
|
||||
}
|
||||
if let Some(num) = it.next() {
|
||||
repl_cut = std::str::from_utf8(num).unwrap().parse::<i32>().unwrap();
|
||||
}
|
||||
}
|
||||
|
||||
// Separate the input pattern into parallel arrays of text (bytes) and digits.
|
||||
let mut got_digit = false;
|
||||
for byte in bytes {
|
||||
if *byte <= b'9' && *byte >= b'0' {
|
||||
assert!(!got_digit, "invalid pattern \"{}\": consecutive digits", pattern);
|
||||
digits.push(*byte);
|
||||
got_digit = true;
|
||||
} else {
|
||||
text.push(*byte);
|
||||
if got_digit {
|
||||
got_digit = false;
|
||||
} else {
|
||||
digits.push(b'0');
|
||||
}
|
||||
}
|
||||
}
|
||||
if !got_digit {
|
||||
digits.push(b'0');
|
||||
}
|
||||
|
||||
if repl_str.is_none() {
|
||||
// Optimize away leading zeroes from the digits array.
|
||||
while digits.len() > 0 && digits[0] == b'0' {
|
||||
digits.remove(0);
|
||||
}
|
||||
} else {
|
||||
// Convert repl_index and repl_cut from Unicode char to byte indexing.
|
||||
let start = if text[0] == b'.' { 1 } else { 0 };
|
||||
if start == 1 {
|
||||
assert_eq!(digits[0], b'0', "unexpected digit before start of word");
|
||||
digits.remove(0);
|
||||
}
|
||||
let word = std::str::from_utf8(&text[start..]).unwrap();
|
||||
let mut chars: Vec<_> = word.char_indices().collect();
|
||||
chars.push((word.len(), '.'));
|
||||
repl_cut = chars[(repl_index + repl_cut) as usize].0 as i32 - chars[repl_index as usize].0 as i32;
|
||||
repl_index = chars[repl_index as usize].0 as i32;
|
||||
}
|
||||
|
||||
// Create the new state, or add pattern into an existing state
|
||||
// (which should not already have a match_string).
|
||||
let mut state_num = self.find_state_number_for(&text);
|
||||
let mut state = &mut self.states[state_num as usize];
|
||||
assert!(state.match_string.is_none(), "duplicate pattern?");
|
||||
if digits.len() > 0 {
|
||||
state.match_string = Some(digits);
|
||||
}
|
||||
if repl_str.is_some() {
|
||||
state.repl_string = repl_str;
|
||||
state.repl_index = repl_index;
|
||||
state.repl_cut = repl_cut;
|
||||
}
|
||||
|
||||
// Set up prefix transitions, inserting additional states as needed.
|
||||
while text.len() > 0 {
|
||||
let last_state = state_num;
|
||||
let ch = *text.last().unwrap();
|
||||
text.truncate(text.len() - 1);
|
||||
state_num = self.find_state_number_for(&text);
|
||||
let exists = self.states[state_num as usize].transitions.0.insert(ch, last_state);
|
||||
if exists.is_some() {
|
||||
assert_eq!(exists.unwrap(), last_state, "overwriting existing transition?");
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn merge_duplicate_states(&mut self) {
|
||||
// We loop here because when we eliminate a duplicate, and update the transitons
|
||||
// that referenced it, we may thereby create new duplicates that another pass
|
||||
// will find and compress further.
|
||||
loop {
|
||||
let orig_len = self.states.len();
|
||||
// Used to map State records to the (first) index at which they occur.
|
||||
let mut state_to_index = HashMap::<&State,i32>::new();
|
||||
// Mapping of old->new state indexes, and whether each old state is
|
||||
// a duplicate that should be dropped.
|
||||
let mut mappings = Vec::<(i32,bool)>::with_capacity(orig_len);
|
||||
let mut next_new_index: i32 = 0;
|
||||
for index in 0 .. self.states.len() {
|
||||
// Find existing index for this state, or allocate the next new index to it.
|
||||
let new_index = *state_to_index.entry(&self.states[index]).or_insert(next_new_index);
|
||||
// Record the mapping, and whether the state was a duplicate.
|
||||
mappings.push((new_index, new_index != next_new_index));
|
||||
// If we used next_new_index for this state, increment it.
|
||||
if new_index == next_new_index {
|
||||
next_new_index += 1;
|
||||
}
|
||||
}
|
||||
// If we didn't find any duplicates, next_new_index will have kept pace with
|
||||
// index, so we know we're finished.
|
||||
if next_new_index as usize == self.states.len() {
|
||||
break;
|
||||
}
|
||||
// Iterate over all the states, either deleting them or updating indexes
|
||||
// according to the mapping we created; then repeat the search.
|
||||
for index in (0 .. self.states.len()).rev() {
|
||||
if mappings[index].1 {
|
||||
self.states.remove(index);
|
||||
} else {
|
||||
let state = &mut self.states[index];
|
||||
if state.fallback_state != -1 {
|
||||
state.fallback_state = mappings[state.fallback_state as usize].0;
|
||||
}
|
||||
for t in state.transitions.0.iter_mut() {
|
||||
*t.1 = mappings[*t.1 as usize].0;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn flatten(&self) -> Vec<u8> {
|
||||
// Calculate total space needed for state data, and build the state_to_offset table.
|
||||
let mut state_data_size = 0;
|
||||
let mut state_to_offset = Vec::<usize>::with_capacity(self.states.len());
|
||||
for state in &self.states {
|
||||
state_to_offset.push(state_data_size);
|
||||
state_data_size += if state.repl_string.is_some() { 12 } else { 8 };
|
||||
state_data_size += state.transitions.0.len() * 4;
|
||||
}
|
||||
|
||||
// Helper to map a state index to its offset in the final data block.
|
||||
let get_state_offset_for = |state_index: i32| -> u32 {
|
||||
if state_index < 0 {
|
||||
return super::INVALID_STATE_OFFSET;
|
||||
}
|
||||
state_to_offset[state_index as usize] as u32
|
||||
};
|
||||
|
||||
// Helper to map a byte string to its offset in the final data block, and
|
||||
// store the bytes into string_data unless using an already-existing string.
|
||||
let mut string_to_offset = HashMap::<Vec<u8>,usize>::new();
|
||||
let mut string_data = Vec::<u8>::new();
|
||||
let mut get_string_offset_for = |bytes: &Option<Vec<u8>>| -> u16 {
|
||||
if bytes.is_none() {
|
||||
return super::INVALID_STRING_OFFSET;
|
||||
}
|
||||
assert!(bytes.as_ref().unwrap().len() < 256);
|
||||
let new_offset = string_data.len();
|
||||
let offset = *string_to_offset.entry(bytes.as_ref().unwrap().clone()).or_insert(new_offset);
|
||||
if offset == new_offset {
|
||||
string_data.push(bytes.as_ref().unwrap().len() as u8);
|
||||
string_data.extend_from_slice(bytes.as_ref().unwrap().as_ref());
|
||||
}
|
||||
offset.try_into().unwrap()
|
||||
};
|
||||
|
||||
// Handle nohyphen string list if present, converting comma separators to NULs
|
||||
// and trimming any surplus whitespace.
|
||||
let mut nohyphen_string_offset: u16 = super::INVALID_STRING_OFFSET;
|
||||
let mut nohyphen_count: u16 = 0;
|
||||
if self.nohyphen.is_some() {
|
||||
let nohyphen_strings: Vec<_> = self.nohyphen.as_ref().unwrap().split(',').map(|x| x.trim()).collect();
|
||||
nohyphen_count = nohyphen_strings.len().try_into().unwrap();
|
||||
nohyphen_string_offset = get_string_offset_for(&Some(nohyphen_strings.join("\0").as_bytes().to_vec()));
|
||||
}
|
||||
|
||||
let mut state_data = Vec::<u8>::with_capacity(state_data_size);
|
||||
for state in &self.states {
|
||||
state_data.extend(&get_state_offset_for(state.fallback_state).to_le_bytes());
|
||||
state_data.extend(&get_string_offset_for(&state.match_string).to_le_bytes());
|
||||
state_data.push(state.transitions.0.len() as u8);
|
||||
// Determine whether to use an extended state record, and if so add the
|
||||
// replacement string and index fields.
|
||||
if state.repl_string.is_none() {
|
||||
state_data.push(0);
|
||||
} else {
|
||||
state_data.push(1);
|
||||
state_data.extend(&get_string_offset_for(&state.repl_string).to_le_bytes());
|
||||
state_data.push(state.repl_index as u8);
|
||||
state_data.push(state.repl_cut as u8);
|
||||
}
|
||||
// Collect transitions into an array so we can sort them.
|
||||
let mut transitions = vec![];
|
||||
for (key, value) in state.transitions.0.iter() {
|
||||
transitions.push((*key, get_state_offset_for(*value)))
|
||||
}
|
||||
transitions.sort();
|
||||
for t in transitions {
|
||||
// New state offset is stored as a 24-bit value, so we do this manually.
|
||||
state_data.push((t.1 & 0xff) as u8);
|
||||
state_data.push(((t.1 >> 8) & 0xff) as u8);
|
||||
state_data.push(((t.1 >> 16) & 0xff) as u8);
|
||||
state_data.push(t.0);
|
||||
}
|
||||
}
|
||||
assert_eq!(state_data.len(), state_data_size);
|
||||
|
||||
// Pad string data to a 4-byte boundary
|
||||
while string_data.len() & 3 != 0 {
|
||||
string_data.push(0);
|
||||
}
|
||||
|
||||
let total_size = super::LEVEL_HEADER_SIZE as usize + state_data_size + string_data.len();
|
||||
let mut result = Vec::<u8>::with_capacity(total_size);
|
||||
|
||||
let state_data_base: u32 = super::LEVEL_HEADER_SIZE as u32;
|
||||
let string_data_base: u32 = state_data_base + state_data_size as u32;
|
||||
|
||||
result.extend(&state_data_base.to_le_bytes());
|
||||
result.extend(&string_data_base.to_le_bytes());
|
||||
result.extend(&nohyphen_string_offset.to_le_bytes());
|
||||
result.extend(&nohyphen_count.to_le_bytes());
|
||||
result.push(self.lh_min);
|
||||
result.push(self.rh_min);
|
||||
result.push(self.clh_min);
|
||||
result.push(self.crh_min);
|
||||
|
||||
result.extend(state_data.iter());
|
||||
result.extend(string_data.iter());
|
||||
|
||||
assert_eq!(result.len(), total_size);
|
||||
|
||||
result
|
||||
}
|
||||
}
|
||||
|
||||
/// Read a libhyphen-style pattern file and create the corresponding state
|
||||
/// machine transitions, etc.
|
||||
/// The returned Vec can be passed to write_hyf_file() to generate a flattened
|
||||
/// representation of the state machine in mapped_hyph's binary format.
|
||||
pub fn read_dic_file<T: Read>(dic_file: T) -> Vec<LevelBuilder> {
|
||||
let reader = BufReader::new(dic_file);
|
||||
|
||||
let mut builders = Vec::<LevelBuilder>::new();
|
||||
builders.push(LevelBuilder::new());
|
||||
let mut builder = &mut builders[0];
|
||||
|
||||
for (index, line) in reader.lines().enumerate() {
|
||||
let mut trimmed = line.unwrap().trim().to_string();
|
||||
// Strip comments.
|
||||
if let Some(i) = trimmed.find('%') {
|
||||
trimmed = trimmed[..i].trim().to_string();
|
||||
}
|
||||
// Ignore empty lines.
|
||||
if trimmed.len() == 0 {
|
||||
continue;
|
||||
}
|
||||
// Uppercase indicates keyword rather than pattern.
|
||||
if trimmed.as_bytes()[0] >= b'A' && trimmed.as_bytes()[0] <= b'Z' {
|
||||
// First line is encoding; we only support UTF-8.
|
||||
if builder.encoding.is_none() {
|
||||
assert_eq!(trimmed, "UTF-8", "Only UTF-8 patterns are accepted!");
|
||||
builder.encoding = Some(trimmed);
|
||||
continue;
|
||||
}
|
||||
// Check for valid keyword-value pairs.
|
||||
if trimmed.contains(' ') {
|
||||
let parts: Vec<&str> = trimmed.split(' ').collect();
|
||||
assert!(parts.len() == 2);
|
||||
let keyword = parts[0];
|
||||
let value = parts[1];
|
||||
match keyword.as_ref() {
|
||||
"LEFTHYPHENMIN" => builder.lh_min = value.parse::<u8>().unwrap(),
|
||||
"RIGHTHYPHENMIN" => builder.rh_min = value.parse::<u8>().unwrap(),
|
||||
"COMPOUNDLEFTHYPHENMIN" => builder.clh_min = value.parse::<u8>().unwrap(),
|
||||
"COMPOUNDRIGHTHYPHENMIN" => builder.crh_min = value.parse::<u8>().unwrap(),
|
||||
"NOHYPHEN" => builder.nohyphen = Some(trimmed),
|
||||
_ => println!("unknown keyword: {}", trimmed),
|
||||
}
|
||||
continue;
|
||||
}
|
||||
// Start a new hyphenation level?
|
||||
if trimmed == "NEXTLEVEL" {
|
||||
builders.push(LevelBuilder::new());
|
||||
builder = builders.last_mut().unwrap();
|
||||
continue;
|
||||
}
|
||||
println!("unknown keyword: {}", trimmed);
|
||||
continue;
|
||||
}
|
||||
// Patterns should always be provided in lowercase; complain if not.
|
||||
assert_eq!(trimmed, trimmed.to_lowercase(), "pattern \"{}\" not lowercased at line {}", trimmed, index);
|
||||
builder.add_pattern(&trimmed);
|
||||
}
|
||||
|
||||
// Create default first (compound-word) level if only one level was provided.
|
||||
// (Maybe this should be optional? Currently just copying libhyphen behavior.)
|
||||
if builders.len() == 1 {
|
||||
let (lh_min, rh_min, clh_min, crh_min) =
|
||||
(builders[0].lh_min, builders[0].rh_min, builders[0].clh_min, builders[0].crh_min);
|
||||
builders.insert(0, LevelBuilder::new());
|
||||
builder = builders.first_mut().unwrap();
|
||||
builder.add_pattern("1-1");
|
||||
builder.add_pattern("1'1");
|
||||
builder.add_pattern("1\u{2013}1"); // en-dash
|
||||
builder.add_pattern("1\u{2019}1"); // curly apostrophe
|
||||
builder.nohyphen = Some("',\u{2013},\u{2019},-".to_string());
|
||||
builder.lh_min = lh_min;
|
||||
builder.rh_min = rh_min;
|
||||
builder.clh_min = if clh_min > 0 { clh_min } else if lh_min > 0 { lh_min } else { 3 };
|
||||
builder.crh_min = if crh_min > 0 { crh_min } else if rh_min > 0 { rh_min } else { 3 };
|
||||
}
|
||||
|
||||
// Put in fallback states in each builder.
|
||||
for builder in &mut builders {
|
||||
for (key, state_index) in builder.str_to_state.iter() {
|
||||
if key.len() == 0 {
|
||||
continue;
|
||||
}
|
||||
let mut fallback_key = key.clone();
|
||||
while fallback_key.len() > 0 {
|
||||
fallback_key.remove(0);
|
||||
if builder.str_to_state.contains_key(&fallback_key) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
builder.states[*state_index as usize].fallback_state = builder.str_to_state[&fallback_key];
|
||||
}
|
||||
}
|
||||
|
||||
// Merge duplicate states to reduce size.
|
||||
for builder in &mut builders {
|
||||
builder.merge_duplicate_states();
|
||||
}
|
||||
|
||||
builders
|
||||
}
|
||||
|
||||
/// Write out the state machines representing a set of hyphenation rules
|
||||
/// to the given output stream.
|
||||
pub fn write_hyf_file<T: Write>(hyf_file: &mut T, levels: Vec<LevelBuilder>) -> std::io::Result<()> {
|
||||
let mut flattened = vec![];
|
||||
for level in levels {
|
||||
flattened.push(level.flatten());
|
||||
}
|
||||
// Write file header: magic number, count of levels.
|
||||
hyf_file.write_all(&[b'H', b'y', b'f', b'0'])?;
|
||||
let level_count: u32 = flattened.len() as u32;
|
||||
hyf_file.write_all(&level_count.to_le_bytes())?;
|
||||
// Write array of offsets to each level. First level will begin immediately
|
||||
// after the array of offsets.
|
||||
let mut offset: u32 = super::FILE_HEADER_SIZE as u32 + 4 * level_count;
|
||||
for flat in &flattened {
|
||||
hyf_file.write_all(&offset.to_le_bytes())?;
|
||||
offset += flat.len() as u32;
|
||||
}
|
||||
// Write the flattened data for each level.
|
||||
for flat in &flattened {
|
||||
hyf_file.write_all(&flat)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
|
@ -0,0 +1,160 @@
|
|||
/* This Source Code Form is subject to the terms of the Mozilla Public
|
||||
* License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
* file, You can obtain one at https://mozilla.org/MPL/2.0/. */
|
||||
|
||||
use std::slice;
|
||||
use std::str;
|
||||
use std::ffi::CStr;
|
||||
use std::os::raw::c_char;
|
||||
use std::str::Utf8Error;
|
||||
|
||||
use memmap::Mmap;
|
||||
|
||||
use super::Hyphenator;
|
||||
|
||||
/// Opaque type representing a hyphenation dictionary loaded from a file,
|
||||
/// for use in FFI function signatures.
|
||||
pub struct HyphDic;
|
||||
|
||||
// Helper to convert word and hyphen buffer parameters from raw C pointer/length
|
||||
// pairs to the Rust types expected by mapped_hyph.
|
||||
unsafe fn params_from_c<'a>(word: *const c_char, word_len: u32,
|
||||
hyphens: *mut u8, hyphens_len: u32) ->
|
||||
(Result<&'a str, Utf8Error>, &'a mut [u8]) {
|
||||
(str::from_utf8(slice::from_raw_parts(word as *const u8, word_len as usize)),
|
||||
slice::from_raw_parts_mut(hyphens, hyphens_len as usize))
|
||||
}
|
||||
|
||||
/// C-callable function to load a hyphenation dictionary from a file at `path`.
|
||||
///
|
||||
/// Returns null on failure.
|
||||
///
|
||||
/// This does not fully validate that the file contains usable hyphenation
|
||||
/// data, it only opens the file (read-only) and mmap's it into memory, and
|
||||
/// does some minimal sanity-checking that it *might* be valid.
|
||||
///
|
||||
/// The returned `HyphDic` must be released with `mapped_hyph_free_dictionary`.
|
||||
///
|
||||
/// # Safety
|
||||
/// The given `path` must be a valid pointer to a NUL-terminated (C-style)
|
||||
/// string.
|
||||
#[no_mangle]
|
||||
pub unsafe extern "C" fn mapped_hyph_load_dictionary(path: *const c_char) -> *const HyphDic {
|
||||
let path_str = match CStr::from_ptr(path).to_str() {
|
||||
Ok(str) => str,
|
||||
Err(_) => return std::ptr::null(),
|
||||
};
|
||||
let hyph = Box::new(match super::load_file(path_str) {
|
||||
Some(dic) => dic,
|
||||
_ => return std::ptr::null(),
|
||||
});
|
||||
Box::into_raw(hyph) as *const HyphDic
|
||||
}
|
||||
|
||||
/// C-callable function to free a hyphenation dictionary
|
||||
/// that was loaded by `mapped_hyph_load_dictionary`.
|
||||
///
|
||||
/// # Safety
|
||||
/// The `dic` parameter must be a `HyphDic` pointer obtained from
|
||||
/// `mapped_hyph_load_dictionary`, and not previously freed.
|
||||
#[no_mangle]
|
||||
pub unsafe extern "C" fn mapped_hyph_free_dictionary(dic: *mut HyphDic) {
|
||||
Box::from_raw(dic);
|
||||
}
|
||||
|
||||
/// C-callable function to find hyphenation values for a given `word`,
|
||||
/// using a dictionary loaded via `mapped_hyph_load_dictionary`.
|
||||
///
|
||||
/// The `word` must be UTF-8-encoded, and is `word_len` bytes (not characters)
|
||||
/// long.
|
||||
///
|
||||
/// Caller must supply the `hyphens` output buffer for results; its size is
|
||||
/// given in `hyphens_len`.
|
||||
/// It should be at least `word_len` elements long.
|
||||
///
|
||||
/// Returns -1 if `word` is not valid UTF-8, or the output `hyphens` buffer is
|
||||
/// too small.
|
||||
/// Otherwise returns the number of potential hyphenation positions found.
|
||||
///
|
||||
/// # Panics
|
||||
/// This function may panic if the given dictionary is not valid.
|
||||
///
|
||||
/// # Safety
|
||||
/// The `dic` parameter must be a `HyphDic` pointer obtained from
|
||||
/// `mapped_hyph_load_dictionary`.
|
||||
///
|
||||
/// The `word` and `hyphens` parameter must be valid pointers to memory buffers
|
||||
/// of at least the respective sizes `word_len` and `hyphens_len`.
|
||||
#[no_mangle]
|
||||
pub unsafe extern "C" fn mapped_hyph_find_hyphen_values_dic(dic: *const HyphDic,
|
||||
word: *const c_char, word_len: u32,
|
||||
hyphens: *mut u8, hyphens_len: u32) -> i32 {
|
||||
if word_len > hyphens_len {
|
||||
return -1;
|
||||
}
|
||||
let (word_str, hyphen_buf) = params_from_c(word, word_len, hyphens, hyphens_len);
|
||||
if word_str.is_err() {
|
||||
return -1;
|
||||
}
|
||||
Hyphenator::new(&*(dic as *const Mmap))
|
||||
.find_hyphen_values(word_str.unwrap(), hyphen_buf) as i32
|
||||
}
|
||||
|
||||
/// C-callable function to find hyphenation values for a given `word`,
|
||||
/// using a dictionary loaded and owned by the caller.
|
||||
///
|
||||
/// The dictionary is supplied as a raw memory buffer `dic_buf` of size
|
||||
/// `dic_len`.
|
||||
///
|
||||
/// The `word` must be UTF-8-encoded, and is `word_len` bytes (not characters)
|
||||
/// long.
|
||||
///
|
||||
/// Caller must supply the `hyphens` output buffer for results; its size is
|
||||
/// given in `hyphens_len`.
|
||||
/// It should be at least `word_len` elements long.
|
||||
///
|
||||
/// Returns -1 if `word` is not valid UTF-8, or the output `hyphens` buffer is
|
||||
/// too small.
|
||||
/// Otherwise returns the number of potential hyphenation positions found.
|
||||
///
|
||||
/// # Panics
|
||||
/// This function may panic if the given dictionary is not valid.
|
||||
///
|
||||
/// # Safety
|
||||
/// The `dic_buf` parameter must be a valid pointer to a memory block of size
|
||||
/// at least `dic_len`.
|
||||
///
|
||||
/// The `word` and `hyphens` parameter must be valid pointers to memory buffers
|
||||
/// of at least the respective sizes `word_len` and `hyphens_len`.
|
||||
#[no_mangle]
|
||||
pub unsafe extern "C" fn mapped_hyph_find_hyphen_values_raw(dic_buf: *const u8, dic_len: u32,
|
||||
word: *const c_char, word_len: u32,
|
||||
hyphens: *mut u8, hyphens_len: u32) -> i32 {
|
||||
if word_len > hyphens_len {
|
||||
return -1;
|
||||
}
|
||||
let (word_str, hyphen_buf) = params_from_c(word, word_len, hyphens, hyphens_len);
|
||||
if word_str.is_err() {
|
||||
return -1;
|
||||
}
|
||||
Hyphenator::new(slice::from_raw_parts(dic_buf, dic_len as usize))
|
||||
.find_hyphen_values(word_str.unwrap(), hyphen_buf) as i32
|
||||
}
|
||||
|
||||
/// C-callable function to check if a given memory buffer `dic_buf` of size
|
||||
/// `dic_len` is potentially usable as a hyphenation dictionary.
|
||||
///
|
||||
/// Returns `true` if the given memory buffer looks like it may be a valid
|
||||
/// hyphenation dictionary, `false` if it is clearly not usable.
|
||||
///
|
||||
/// # Safety
|
||||
/// The `dic_buf` parameter must be a valid pointer to a memory block of size
|
||||
/// at least `dic_len`.
|
||||
#[no_mangle]
|
||||
pub unsafe extern "C" fn mapped_hyph_is_valid_hyphenator(dic_buf: *const u8, dic_len: u32) -> bool {
|
||||
if dic_buf.is_null() {
|
||||
return false;
|
||||
}
|
||||
let dic = Hyphenator::new(slice::from_raw_parts(dic_buf, dic_len as usize));
|
||||
dic.is_valid_hyphenator()
|
||||
}
|
||||
|
|
@ -0,0 +1,625 @@
|
|||
/* This Source Code Form is subject to the terms of the Mozilla Public
|
||||
* License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
* file, You can obtain one at https://mozilla.org/MPL/2.0/. */
|
||||
|
||||
#[macro_use]
|
||||
extern crate arrayref;
|
||||
extern crate memmap;
|
||||
|
||||
use std::slice;
|
||||
use std::str;
|
||||
use std::cmp::max;
|
||||
use std::fs::File;
|
||||
use std::mem::size_of;
|
||||
|
||||
use memmap::Mmap;
|
||||
|
||||
// Make submodules available publicly.
|
||||
pub mod builder;
|
||||
pub mod ffi;
|
||||
|
||||
// 4-byte identification expected at beginning of a compiled dictionary file.
|
||||
// (This will be updated if an incompatible change to the format is made in
|
||||
// some future revision.)
|
||||
const MAGIC_NUMBER: [u8; 4] = [b'H', b'y', b'f', b'0'];
|
||||
|
||||
const INVALID_STRING_OFFSET: u16 = 0xffff;
|
||||
const INVALID_STATE_OFFSET: u32 = 0xffffff;
|
||||
|
||||
const FILE_HEADER_SIZE: usize = 8; // 4-byte magic number, 4-byte count of levels
|
||||
const LEVEL_HEADER_SIZE: usize = 16;
|
||||
const STATE_HEADER_SIZE_BASIC: usize = 8;
|
||||
const STATE_HEADER_SIZE_EXTENDED: usize = 12;
|
||||
|
||||
// Transition actually holds a 24-bit new state offset and an 8-bit input byte
|
||||
// to match. We will be interpreting byte ranges as Transition arrays (in the
|
||||
// State::transitions() method below), so use repr(C) to ensure we have the
|
||||
// memory layout we expect.
|
||||
// Transition records do not depend on any specific alignment.
|
||||
#[repr(C)]
|
||||
#[derive(Debug,Copy,Clone)]
|
||||
struct Transition(u8, u8, u8, u8);
|
||||
|
||||
impl Transition {
|
||||
fn new_state_offset(&self) -> usize {
|
||||
// Read a 24-bit little-endian number from three bytes.
|
||||
self.0 as usize + ((self.1 as usize) << 8) + ((self.2 as usize) << 16)
|
||||
}
|
||||
fn match_byte(&self) -> u8 {
|
||||
self.3
|
||||
}
|
||||
}
|
||||
|
||||
// State is an area of the Level's data block that begins with a fixed header,
|
||||
// followed by an array of transitions. The total size of each State's data
|
||||
// depends on the number of transitions in the state. Only the basic header
|
||||
// is defined by the struct here; the rest of the state is accessed via
|
||||
// pointer magic.
|
||||
// There are two versions of State, a basic version that supports only simple
|
||||
// hyphenation (no associated spelling change), and an extended version that
|
||||
// adds the replacement-string fields to support spelling changes at the
|
||||
// hyphenation point. Check is_extended() to know which version is present.
|
||||
// State records are NOT necessarily 4-byte aligned, so multi-byte fields
|
||||
// should be read with care.
|
||||
#[derive(Debug,Copy,Clone)]
|
||||
#[repr(C)]
|
||||
struct State {
|
||||
fallback_state_: [u8; 4],
|
||||
match_string_offset_: [u8; 2],
|
||||
num_transitions_: u8,
|
||||
is_extended_: u8,
|
||||
}
|
||||
|
||||
impl State {
|
||||
// Accessors for the various State header fields; see file format description.
|
||||
fn fallback_state(&self) -> usize {
|
||||
u32::from_le_bytes(self.fallback_state_) as usize
|
||||
}
|
||||
fn match_string_offset(&self) -> usize {
|
||||
u16::from_le_bytes(self.match_string_offset_) as usize
|
||||
}
|
||||
fn num_transitions(&self) -> u8 {
|
||||
self.num_transitions_
|
||||
}
|
||||
fn is_extended(&self) -> bool {
|
||||
self.is_extended_ != 0
|
||||
}
|
||||
// Accessors that are only valid if is_extended() is true.
|
||||
// These use `unsafe` to dereference a pointer to the relevant field;
|
||||
// this is OK because Level::get_state always validates the total state size
|
||||
// before returning a state reference, so these pointers will be valid for
|
||||
// any extended state it returns.
|
||||
#[allow(dead_code)]
|
||||
fn repl_string_offset(&self) -> usize {
|
||||
debug_assert!(self.is_extended());
|
||||
u16::from_le(unsafe { *((self as *const State as *const u8).offset(8) as *const u16) }) as usize
|
||||
}
|
||||
#[allow(dead_code)]
|
||||
fn repl_index(&self) -> i8 {
|
||||
debug_assert!(self.is_extended());
|
||||
unsafe { *((self as *const State as *const i8).offset(10)) }
|
||||
}
|
||||
#[allow(dead_code)]
|
||||
fn repl_cut(&self) -> i8 {
|
||||
debug_assert!(self.is_extended());
|
||||
unsafe { *((self as *const State as *const i8).offset(11)) }
|
||||
}
|
||||
// Return the state's Transitions as a slice reference.
|
||||
fn transitions(&self) -> &[Transition] {
|
||||
let count = self.num_transitions() as usize;
|
||||
if count == 0 {
|
||||
return &[];
|
||||
}
|
||||
let transition_offset = if self.is_extended() { STATE_HEADER_SIZE_EXTENDED } else { STATE_HEADER_SIZE_BASIC } as isize;
|
||||
// We know the `offset` here will not look beyond the valid range of memory
|
||||
// because Level::get_state() checks the state length (accounting for the
|
||||
// number of transitions) before returning a State reference.
|
||||
let trans_ptr = unsafe { (self as *const State as *const u8).offset(transition_offset) as *const Transition };
|
||||
// Again, because Level::get_state() already checked the state length, we know
|
||||
// this slice address and count will be valid.
|
||||
unsafe { slice::from_raw_parts(trans_ptr, count) }
|
||||
}
|
||||
// Look up the Transition for a given input byte, or None.
|
||||
fn transition_for(&self, b: u8) -> Option<Transition> {
|
||||
// The transitions array is sorted by match_byte() value, but there are
|
||||
// usually very few entries; benchmarking showed that using binary_search_by
|
||||
// here gave no benefit (possibly slightly slower).
|
||||
self.transitions().iter().copied().find(|t| t.match_byte() == b)
|
||||
}
|
||||
// Just for debugging use...
|
||||
#[allow(dead_code)]
|
||||
fn deep_show(&self, prefix: &str, dic: &Level) {
|
||||
if self.match_string_offset() != INVALID_STRING_OFFSET as usize {
|
||||
let match_string = dic.string_at_offset(self.match_string_offset());
|
||||
println!("{}match: {}", prefix, str::from_utf8(match_string).unwrap());
|
||||
}
|
||||
for t in self.transitions() {
|
||||
println!("{}{} ->", prefix, t.match_byte() as char);
|
||||
let next_prefix = format!("{} ", prefix);
|
||||
dic.get_state(t.new_state_offset()).unwrap().deep_show(&next_prefix, &dic);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// We count the presentation-form ligature characters U+FB00..FB06 as multiple
|
||||
// chars for the purposes of lefthyphenmin/righthyphenmin. In UTF-8, all these
|
||||
// ligature characters are 3-byte sequences beginning with <0xEF, 0xAC>; this
|
||||
// helper returns the "decomposed length" of the ligature given its trailing
|
||||
// byte.
|
||||
fn lig_length(trail_byte: u8) -> usize {
|
||||
// This is only called on valid UTF-8 where we already know trail_byte
|
||||
// must be >= 0x80.
|
||||
// Ligature lengths: ff fi fl ffi ffl long-st st
|
||||
const LENGTHS: [u8; 7] = [ 2u8, 2u8, 2u8, 3u8, 3u8, 2u8, 2u8 ];
|
||||
if trail_byte > 0x86 {
|
||||
return 1;
|
||||
}
|
||||
LENGTHS[trail_byte as usize - 0x80] as usize
|
||||
}
|
||||
|
||||
fn is_utf8_trail_byte(byte: u8) -> bool {
|
||||
(byte & 0xC0) == 0x80
|
||||
}
|
||||
|
||||
fn is_ascii_digit(byte: u8) -> bool {
|
||||
byte <= b'9' && byte >= b'0'
|
||||
}
|
||||
|
||||
fn is_odd(byte: u8) -> bool {
|
||||
(byte & 0x01) == 0x01
|
||||
}
|
||||
|
||||
// A hyphenation Level has a header followed by State records and packed string
|
||||
// data. The total size of the slice depends on the number and size of the
|
||||
// States and Strings it contains.
|
||||
// Note that the data of the Level may not have any specific alignment!
|
||||
#[derive(Debug,Copy,Clone)]
|
||||
struct Level<'a> {
|
||||
data: &'a [u8],
|
||||
// Header fields cached by the constructor for faster access:
|
||||
state_data_base_: usize,
|
||||
string_data_base_: usize,
|
||||
}
|
||||
|
||||
impl Level<'_> {
|
||||
// Constructor that initializes our cache variables.
|
||||
fn new(data: &[u8]) -> Level {
|
||||
Level {
|
||||
data: data,
|
||||
state_data_base_: u32::from_le_bytes(*array_ref!(data, 0, 4)) as usize,
|
||||
string_data_base_: u32::from_le_bytes(*array_ref!(data, 4, 4)) as usize,
|
||||
}
|
||||
}
|
||||
|
||||
// Accessors for Level header fields; see file format description.
|
||||
fn state_data_base(&self) -> usize {
|
||||
self.state_data_base_ // cached by constructor
|
||||
}
|
||||
fn string_data_base(&self) -> usize {
|
||||
self.string_data_base_ // cached by constructor
|
||||
}
|
||||
fn nohyphen_string_offset(&self) -> usize {
|
||||
u16::from_le_bytes(*array_ref!(self.data, 8, 2)) as usize
|
||||
}
|
||||
#[allow(dead_code)]
|
||||
fn nohyphen_count(&self) -> u16 {
|
||||
u16::from_le_bytes(*array_ref!(self.data, 10, 2))
|
||||
}
|
||||
fn lh_min(&self) -> usize {
|
||||
max(1, self.data[12] as usize)
|
||||
}
|
||||
fn rh_min(&self) -> usize {
|
||||
max(1, self.data[13] as usize)
|
||||
}
|
||||
fn clh_min(&self) -> usize {
|
||||
max(1, self.data[14] as usize)
|
||||
}
|
||||
fn crh_min(&self) -> usize {
|
||||
max(1, self.data[15] as usize)
|
||||
}
|
||||
fn word_boundary_mins(&self) -> (usize, usize, usize, usize) {
|
||||
(self.lh_min(), self.rh_min(), self.clh_min(), self.crh_min())
|
||||
}
|
||||
// Strings are represented as offsets from the Level's string_data_base.
|
||||
// This returns a byte slice referencing the string at a given offset,
|
||||
// or an empty slice if invalid.
|
||||
fn string_at_offset(&self, offset: usize) -> &'_ [u8] {
|
||||
if offset == INVALID_STRING_OFFSET as usize {
|
||||
return &[];
|
||||
}
|
||||
let string_base = self.string_data_base() as usize + offset;
|
||||
// TODO: move this to the validation function.
|
||||
debug_assert!(string_base + 1 <= self.data.len());
|
||||
if string_base + 1 > self.data.len() {
|
||||
return &[];
|
||||
}
|
||||
let len = self.data[string_base] as usize;
|
||||
// TODO: move this to the validation function.
|
||||
debug_assert!(string_base + 1 + len <= self.data.len());
|
||||
if string_base + 1 + len > self.data.len() {
|
||||
return &[];
|
||||
}
|
||||
self.data.get(string_base + 1 .. string_base + 1 + len).unwrap()
|
||||
}
|
||||
// The nohyphen field actually contains multiple NUL-separated substrings;
|
||||
// return them as a vector of individual byte slices.
|
||||
fn nohyphen(&self) -> Vec<&[u8]> {
|
||||
let string_offset = self.nohyphen_string_offset();
|
||||
let nohyph_str = self.string_at_offset(string_offset as usize);
|
||||
if nohyph_str.len() == 0 {
|
||||
return vec![];
|
||||
}
|
||||
nohyph_str.split(|&b| b == 0).collect()
|
||||
}
|
||||
// States are represented as an offset from the Level's state_data_base.
|
||||
// This returns a reference to the State at a given offset, or None if invalid.
|
||||
fn get_state(&self, offset: usize) -> Option<&State> {
|
||||
if offset == INVALID_STATE_OFFSET as usize {
|
||||
return None;
|
||||
}
|
||||
debug_assert_eq!(offset & 3, 0);
|
||||
let state_base = self.state_data_base() + offset;
|
||||
// TODO: move this to the validation function.
|
||||
debug_assert!(state_base + STATE_HEADER_SIZE_BASIC <= self.string_data_base());
|
||||
if state_base + STATE_HEADER_SIZE_BASIC > self.string_data_base() {
|
||||
return None;
|
||||
}
|
||||
let state_ptr = &self.data[state_base] as *const u8 as *const State;
|
||||
// This is safe because we just checked against self.string_data_base() above.
|
||||
let state = unsafe { state_ptr.as_ref().unwrap() };
|
||||
let length = if state.is_extended() { STATE_HEADER_SIZE_EXTENDED } else { STATE_HEADER_SIZE_BASIC }
|
||||
+ size_of::<Transition>() * state.num_transitions() as usize;
|
||||
// TODO: move this to the validation function.
|
||||
debug_assert!(state_base + length <= self.string_data_base());
|
||||
if state_base + length > self.string_data_base() {
|
||||
return None;
|
||||
}
|
||||
// This is safe because we checked the full state length against self.string_data_base().
|
||||
unsafe { state_ptr.as_ref() }
|
||||
}
|
||||
// Sets hyphenation values (odd = potential break, even = no break) in values[],
|
||||
// and returns the change in the number of odd values present, so the caller can
|
||||
// keep track of the total number of potential breaks in the word.
|
||||
fn find_hyphen_values(&self, word: &str, values: &mut [u8], lh_min: usize, rh_min: usize) -> isize {
|
||||
// Bail out immediately if the word is too short to hyphenate.
|
||||
if word.len() < lh_min + rh_min {
|
||||
return 0;
|
||||
}
|
||||
let start_state = self.get_state(0);
|
||||
let mut st = start_state;
|
||||
let mut hyph_count = 0;
|
||||
for i in 0 .. word.len() + 2 {
|
||||
// Loop over the word by bytes, with a virtual '.' added at each end
|
||||
// to match word-boundary patterns.
|
||||
let b = if i == 0 || i == word.len() + 1 { b'.' } else { word.as_bytes()[i - 1] };
|
||||
loop {
|
||||
// Loop to repeatedly fall back if we don't find a matching transition.
|
||||
// Note that this could infinite-loop if there is a state whose fallback
|
||||
// points to itself (or a cycle of fallbacks), but this would represent
|
||||
// a table compilation error.
|
||||
// (A potential validation function could check for fallback cycles.)
|
||||
if st.is_none() {
|
||||
st = start_state;
|
||||
break;
|
||||
}
|
||||
let state = st.unwrap();
|
||||
if let Some(tr) = state.transition_for(b) {
|
||||
// Found a transition for the current byte. Look up the new state;
|
||||
// if it has a match_string, merge its weights into `values`.
|
||||
st = self.get_state(tr.new_state_offset());
|
||||
if let Some(state) = st {
|
||||
let match_offset = state.match_string_offset();
|
||||
if match_offset != INVALID_STRING_OFFSET as usize {
|
||||
if state.is_extended() {
|
||||
debug_assert!(false, "extended hyphenation not supported by this function");
|
||||
} else {
|
||||
let match_str = self.string_at_offset(match_offset);
|
||||
let offset = i + 1 - match_str.len();
|
||||
assert!(offset + match_str.len() <= word.len() + 2);
|
||||
for j in 0 .. match_str.len() {
|
||||
let index = offset + j;
|
||||
if index >= lh_min && index <= word.len() - rh_min {
|
||||
// lh_min and rh_min are guaranteed to be >= 1,
|
||||
// so this will not try to access outside values[].
|
||||
let old_value = values[index - 1];
|
||||
let value = match_str[j] - b'0';
|
||||
if value > old_value {
|
||||
if is_odd(old_value) != is_odd(value) {
|
||||
// Adjust hyph_count for the change we're making
|
||||
hyph_count += if is_odd(value) { 1 } else { -1 };
|
||||
}
|
||||
values[index - 1] = value;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
// We have handled the current input byte; leave the fallback loop
|
||||
// and get next input.
|
||||
break;
|
||||
}
|
||||
// No transition for the current byte; go to fallback state and try again.
|
||||
st = self.get_state(state.fallback_state());
|
||||
}
|
||||
}
|
||||
|
||||
// If the word was not purely ASCII, or if the word begins/ends with
|
||||
// digits, the use of lh_min and rh_min above may not have correctly
|
||||
// excluded enough positions, so we need to fix things up here.
|
||||
let mut index = 0;
|
||||
let mut count = 0;
|
||||
let word_bytes = word.as_bytes();
|
||||
let mut clear_hyphen_at = |i| { if is_odd(values[i]) { hyph_count -= 1; } values[i] = 0; };
|
||||
// Handle lh_min.
|
||||
while count < lh_min - 1 && index < word_bytes.len() {
|
||||
let byte = word_bytes[index];
|
||||
clear_hyphen_at(index);
|
||||
if byte < 0x80 {
|
||||
index += 1;
|
||||
if is_ascii_digit(byte) {
|
||||
continue; // ASCII digits don't count
|
||||
}
|
||||
} else if byte == 0xEF && word_bytes[index + 1] == 0xAC {
|
||||
// Unicode presentation-form ligature characters, which we count as
|
||||
// multiple chars for the purpose of lh_min/rh_min, all begin with
|
||||
// 0xEF, 0xAC in UTF-8.
|
||||
count += lig_length(word_bytes[index + 2]);
|
||||
clear_hyphen_at(index + 1);
|
||||
clear_hyphen_at(index + 2);
|
||||
index += 3;
|
||||
continue;
|
||||
} else {
|
||||
index += 1;
|
||||
while index < word_bytes.len() && is_utf8_trail_byte(word_bytes[index]) {
|
||||
clear_hyphen_at(index);
|
||||
index += 1;
|
||||
}
|
||||
}
|
||||
count += 1;
|
||||
}
|
||||
|
||||
// Handle rh_min.
|
||||
count = 0;
|
||||
index = word.len();
|
||||
while count < rh_min && index > 0 {
|
||||
index -= 1;
|
||||
let byte = word_bytes[index];
|
||||
if index < word.len() - 1 {
|
||||
clear_hyphen_at(index);
|
||||
}
|
||||
if byte < 0x80 {
|
||||
// Only count if not an ASCII digit
|
||||
if !is_ascii_digit(byte) {
|
||||
count += 1;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if is_utf8_trail_byte(byte) {
|
||||
continue;
|
||||
}
|
||||
if byte == 0xEF && word_bytes[index + 1] == 0xAC {
|
||||
// Presentation-form ligatures count as multiple chars.
|
||||
count += lig_length(word_bytes[index + 2]);
|
||||
continue;
|
||||
}
|
||||
count += 1;
|
||||
}
|
||||
|
||||
hyph_count
|
||||
}
|
||||
}
|
||||
|
||||
/// Hyphenation engine encapsulating a language-specific set of patterns (rules)
|
||||
/// that identify possible break positions within a word.
|
||||
pub struct Hyphenator<'a>(&'a [u8]);
|
||||
|
||||
impl Hyphenator<'_> {
|
||||
/// Return a Hyphenator that wraps the given buffer.
|
||||
/// This does *not* check that the given buffer is in fact a valid hyphenation table.
|
||||
/// Use is_valid_hyphenator() to determine whether it is usable.
|
||||
/// (Calling hyphenation methods on a Hyphenator that wraps arbitrary,
|
||||
/// unvalidated data is not unsafe, but may panic.)
|
||||
pub fn new(buffer: &[u8]) -> Hyphenator {
|
||||
Hyphenator(buffer)
|
||||
}
|
||||
|
||||
// Internal implementation details
|
||||
fn magic_number(&self) -> &[u8] {
|
||||
&self.0[0 .. 4]
|
||||
}
|
||||
fn num_levels(&self) -> usize {
|
||||
u32::from_le_bytes(*array_ref!(self.0, 4, 4)) as usize
|
||||
}
|
||||
fn level(&self, i: usize) -> Level {
|
||||
let offset = u32::from_le_bytes(*array_ref!(self.0, FILE_HEADER_SIZE + 4 * i, 4)) as usize;
|
||||
let limit = if i == self.num_levels() - 1 {
|
||||
self.0.len()
|
||||
} else {
|
||||
u32::from_le_bytes(*array_ref!(self.0, FILE_HEADER_SIZE + 4 * i + 4, 4)) as usize
|
||||
};
|
||||
debug_assert!(offset + LEVEL_HEADER_SIZE <= limit && limit <= self.0.len());
|
||||
debug_assert_eq!(offset & 3, 0);
|
||||
debug_assert_eq!(limit & 3, 0);
|
||||
Level::new(&self.0[offset .. limit])
|
||||
}
|
||||
|
||||
/// Identify acceptable hyphenation positions in the given `word`.
|
||||
///
|
||||
/// The caller-supplied `values` must be at least as long as the `word`.
|
||||
///
|
||||
/// On return, any elements with an odd value indicate positions in the word
|
||||
/// after which a hyphen could be inserted.
|
||||
///
|
||||
/// Returns the number of possible hyphenation positions that were found.
|
||||
///
|
||||
/// # Panics
|
||||
/// If the given `values` slice is too small to hold the results.
|
||||
///
|
||||
/// If the block of memory represented by `self.0` is not in fact a valid
|
||||
/// hyphenation dictionary, this function may panic with an overflow or
|
||||
/// array bounds violation.
|
||||
pub fn find_hyphen_values(&self, word: &str, values: &mut [u8]) -> isize {
|
||||
assert!(values.len() >= word.len());
|
||||
values.iter_mut().for_each(|x| *x = 0);
|
||||
let top_level = self.level(0);
|
||||
let (lh_min, rh_min, clh_min, crh_min) = top_level.word_boundary_mins();
|
||||
if word.len() < lh_min + rh_min {
|
||||
return 0;
|
||||
}
|
||||
let mut hyph_count = top_level.find_hyphen_values(word, values, lh_min, rh_min);
|
||||
let compound = hyph_count > 0;
|
||||
// Subsequent levels are applied to fragments between potential breaks
|
||||
// already found:
|
||||
for l in 1 .. self.num_levels() {
|
||||
let level = self.level(l);
|
||||
if hyph_count > 0 {
|
||||
let mut begin = 0;
|
||||
let mut lh = lh_min;
|
||||
// lh_min and rh_min are both guaranteed to be greater than zero,
|
||||
// so this loop will not reach fully to the end of the word.
|
||||
for i in lh_min - 1 .. word.len() - rh_min {
|
||||
if is_odd(values[i]) {
|
||||
if i > begin {
|
||||
// We've found a component of a compound;
|
||||
// clear the corresponding values and apply the new level.
|
||||
// (These values must be even, so hyph_count is unchanged.)
|
||||
values[begin .. i].iter_mut().for_each(|x| {
|
||||
*x = 0;
|
||||
});
|
||||
hyph_count += level.find_hyphen_values(&word[begin .. i + 1],
|
||||
&mut values[begin .. i + 1],
|
||||
lh, crh_min);
|
||||
}
|
||||
begin = i + 1;
|
||||
lh = clh_min;
|
||||
}
|
||||
}
|
||||
if begin == 0 {
|
||||
// No compound-word breaks were found, just apply level to the whole word.
|
||||
hyph_count += level.find_hyphen_values(word, values, lh_min, rh_min);
|
||||
} else if begin < word.len() {
|
||||
// Handle trailing component of compound.
|
||||
hyph_count += level.find_hyphen_values(&word[begin .. word.len()],
|
||||
&mut values[begin .. word.len()],
|
||||
clh_min, rh_min);
|
||||
}
|
||||
} else {
|
||||
hyph_count += level.find_hyphen_values(word, values, lh_min, rh_min);
|
||||
}
|
||||
}
|
||||
|
||||
// Only need to check nohyphen strings if top-level (compound) breaks were found.
|
||||
if compound && hyph_count > 0 {
|
||||
let nohyph = top_level.nohyphen();
|
||||
if nohyph.len() > 0 {
|
||||
for i in lh_min .. word.len() - rh_min + 1 {
|
||||
if is_odd(values[i - 1]) {
|
||||
for nh in &nohyph {
|
||||
if i + nh.len() <= word.len() && *nh == &word.as_bytes()[i .. i + nh.len()] {
|
||||
values[i - 1] = 0;
|
||||
hyph_count -= 1;
|
||||
break;
|
||||
}
|
||||
if nh.len() <= i && *nh == &word.as_bytes()[i - nh.len() .. i] {
|
||||
values[i - 1] = 0;
|
||||
hyph_count -= 1;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
hyph_count
|
||||
}
|
||||
|
||||
/// Generate the hyphenated form of a `word` by inserting the given `hyphen_char`
|
||||
/// at each valid break position.
|
||||
///
|
||||
/// # Panics
|
||||
/// If the block of memory represented by `self` is not in fact a valid
|
||||
/// hyphenation dictionary, this function may panic with an overflow or
|
||||
/// array bounds violation.
|
||||
///
|
||||
/// Also panics if the length of the hyphenated word would overflow `usize`.
|
||||
pub fn hyphenate_word(&self, word: &str, hyphchar: char) -> String {
|
||||
let mut values = vec![0u8; word.len()];
|
||||
let hyph_count = self.find_hyphen_values(word, &mut values);
|
||||
if hyph_count <= 0 {
|
||||
return word.to_string();
|
||||
}
|
||||
// We know how long the result will be, so we can preallocate here.
|
||||
let result_len = word.len() + hyph_count as usize * hyphchar.len_utf8();
|
||||
let mut result = String::with_capacity(result_len);
|
||||
let mut n = 0;
|
||||
for ch in word.char_indices() {
|
||||
if ch.0 > 0 && is_odd(values[ch.0 - 1]) {
|
||||
result.push(hyphchar);
|
||||
n += 1;
|
||||
}
|
||||
result.push(ch.1);
|
||||
}
|
||||
debug_assert_eq!(n, hyph_count);
|
||||
debug_assert_eq!(result_len, result.len());
|
||||
result
|
||||
}
|
||||
|
||||
/// Check if the block of memory looks like it could be a valid hyphenation
|
||||
/// table.
|
||||
pub fn is_valid_hyphenator(&self) -> bool {
|
||||
// Size must be at least 4 bytes for magic_number + 4 bytes num_levels;
|
||||
// smaller than this cannot be safely inspected.
|
||||
if self.0.len() < FILE_HEADER_SIZE {
|
||||
return false;
|
||||
}
|
||||
if self.magic_number() != MAGIC_NUMBER {
|
||||
return false;
|
||||
}
|
||||
// For each level, there's a 4-byte offset in the header, and the level
|
||||
// has its own 16-byte header, so we can check a minimum size again here.
|
||||
let num_levels = self.num_levels();
|
||||
if self.0.len() < FILE_HEADER_SIZE + LEVEL_HEADER_SIZE * num_levels {
|
||||
return false;
|
||||
}
|
||||
// Check that state_data_base and string_data_base for each hyphenation
|
||||
// level are within range.
|
||||
for l in 0 .. num_levels {
|
||||
let level = self.level(l);
|
||||
if level.state_data_base() < LEVEL_HEADER_SIZE ||
|
||||
level.state_data_base() > level.string_data_base() ||
|
||||
level.string_data_base() > level.data.len() {
|
||||
return false;
|
||||
}
|
||||
// TODO: consider doing more extensive validation of states and
|
||||
// strings within the level?
|
||||
}
|
||||
// It's still possible the dic is internally broken, but at least it's
|
||||
// worth trying to use it!
|
||||
true
|
||||
}
|
||||
}
|
||||
|
||||
/// Load the compiled hyphenation file at `dic_path`, if present.
|
||||
///
|
||||
/// Returns `None` if the specified file cannot be opened or mapped,
|
||||
/// otherwise returns a `memmap::Mmap` mapping the file.
|
||||
///
|
||||
/// This is unsafe for the same reason Mmap::map() is unsafe:
|
||||
/// mapped_hyph does not guarantee safety if the mapped file is modified
|
||||
/// (e.g. by another process) while we're using it.
|
||||
///
|
||||
/// This verifies that the file looks superficially like it may be a
|
||||
/// compiled hyphenation table, but does *not* fully check the validity
|
||||
/// of the file contents! Calling hyphenation functions with the returned
|
||||
/// data is not unsafe, but may panic if the data is invalid.
|
||||
pub unsafe fn load_file(dic_path: &str) -> Option<Mmap> {
|
||||
let file = File::open(dic_path).ok()?;
|
||||
let dic = Mmap::map(&file).ok()?;
|
||||
let hyph = Hyphenator(&*dic);
|
||||
if hyph.is_valid_hyphenator() {
|
||||
return Some(dic);
|
||||
}
|
||||
None
|
||||
}
|
||||
|
|
@ -0,0 +1,62 @@
|
|||
/* This Source Code Form is subject to the terms of the Mozilla Public
|
||||
* License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
* file, You can obtain one at https://mozilla.org/MPL/2.0/. */
|
||||
|
||||
extern crate mapped_hyph;
|
||||
|
||||
use mapped_hyph::Hyphenator;
|
||||
|
||||
fn main() {
|
||||
let dic_path = "hyph_en_US.hyf";
|
||||
|
||||
let dic = match unsafe { mapped_hyph::load_file(dic_path) } {
|
||||
Some(dic) => dic,
|
||||
_ => panic!("failed to load dictionary {}", dic_path),
|
||||
};
|
||||
let hyph = Hyphenator::new(&*dic);
|
||||
|
||||
println!("{}", hyph.hyphenate_word("haha", '-'));
|
||||
println!("{}", hyph.hyphenate_word("hahaha", '-'));
|
||||
println!("{}", hyph.hyphenate_word("photo", '-'));
|
||||
println!("{}", hyph.hyphenate_word("photograph", '-'));
|
||||
println!("{}", hyph.hyphenate_word("photographer", '-'));
|
||||
println!("{}", hyph.hyphenate_word("photographic", '-'));
|
||||
println!("{}", hyph.hyphenate_word("photographical", '-'));
|
||||
println!("{}", hyph.hyphenate_word("photographically", '-'));
|
||||
println!("{}", hyph.hyphenate_word("supercalifragilisticexpialidocious", '-'));
|
||||
println!("{}", hyph.hyphenate_word("o'dwyer", '='));
|
||||
println!("{}", hyph.hyphenate_word("o'callahan", '='));
|
||||
println!("{}", hyph.hyphenate_word("o’dwyer", '='));
|
||||
println!("{}", hyph.hyphenate_word("o’callahan", '='));
|
||||
println!("{}", hyph.hyphenate_word("petti-fogging", '='));
|
||||
println!("{}", hyph.hyphenate_word("e-mailing", '='));
|
||||
println!("{}", hyph.hyphenate_word("-x-mailing", '='));
|
||||
println!("{}", hyph.hyphenate_word("-strikeout-", '='));
|
||||
|
||||
let dic2 = match unsafe { mapped_hyph::load_file("tests/compound.hyf") } {
|
||||
Some(dic) => dic,
|
||||
_ => panic!("failed to load dictionary {}", "tests/compound.hyf"),
|
||||
};
|
||||
|
||||
let h2 = Hyphenator::new(&*dic2);
|
||||
println!("{}", h2.hyphenate_word("motorcycle", '='));
|
||||
|
||||
let dic3 = match unsafe { mapped_hyph::load_file("tests/rhmin.hyf") } {
|
||||
Some(dic) => dic,
|
||||
_ => panic!("failed to load dictionary {}", dic_path),
|
||||
};
|
||||
let h3 = Hyphenator::new(&*dic3);
|
||||
println!("{}", h3.hyphenate_word("övéit", '='));
|
||||
println!("{}", h3.hyphenate_word("అంగడిధర", '='));
|
||||
|
||||
let dic4 = match unsafe { mapped_hyph::load_file("tests/num.hyf") } {
|
||||
Some(dic) => dic,
|
||||
_ => panic!("failed to load dictionary {}", "tests/num.hyf"),
|
||||
};
|
||||
let h4 = Hyphenator::new(&*dic4);
|
||||
|
||||
println!("{}", h4.hyphenate_word("123foobar123", '='));
|
||||
println!("{}", h4.hyphenate_word("123foobarfoobar", '='));
|
||||
println!("{}", h4.hyphenate_word("foobarfoobar123", '='));
|
||||
println!("{}", h4.hyphenate_word("123foobarfoobar123", '='));
|
||||
}
|
||||
Двоичный файл не отображается.
Разница между файлами не показана из-за своего большого размера
Загрузить разницу
Разница между файлами не показана из-за своего большого размера
Загрузить разницу
Двоичный файл не отображается.
Двоичный файл не отображается.
Двоичный файл не отображается.
Двоичный файл не отображается.
Двоичный файл не отображается.
Двоичный файл не отображается.
Двоичный файл не отображается.
Двоичный файл не отображается.
Двоичный файл не отображается.
Двоичный файл не отображается.
|
|
@ -0,0 +1,178 @@
|
|||
/* This Source Code Form is subject to the terms of the Mozilla Public
|
||||
* License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
* file, You can obtain one at https://mozilla.org/MPL/2.0/. */
|
||||
|
||||
#![feature(test)]
|
||||
|
||||
extern crate test;
|
||||
extern crate mapped_hyph;
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use mapped_hyph::Hyphenator;
|
||||
|
||||
#[test]
|
||||
fn basic_tests() {
|
||||
let dic_path = "hyph_en_US.hyf";
|
||||
let dic = match unsafe { mapped_hyph::load_file(dic_path) } {
|
||||
Some(dic) => dic,
|
||||
_ => panic!("failed to load dictionary {}", dic_path),
|
||||
};
|
||||
let hyph = Hyphenator::new(&*dic);
|
||||
assert_eq!(hyph.hyphenate_word("haha", '-'), "haha");
|
||||
assert_eq!(hyph.hyphenate_word("hahaha", '-'), "ha-haha");
|
||||
assert_eq!(hyph.hyphenate_word("photo", '-'), "photo");
|
||||
assert_eq!(hyph.hyphenate_word("photograph", '-'), "pho-to-graph");
|
||||
assert_eq!(hyph.hyphenate_word("photographer", '-'), "pho-tog-ra-pher");
|
||||
assert_eq!(hyph.hyphenate_word("photographic", '-'), "pho-to-graphic");
|
||||
assert_eq!(hyph.hyphenate_word("photographical", '-'), "pho-to-graph-i-cal");
|
||||
assert_eq!(hyph.hyphenate_word("photographically", '-'), "pho-to-graph-i-cally");
|
||||
assert_eq!(hyph.hyphenate_word("supercalifragilisticexpialidocious", '-'), "su-per-cal-ifrag-ilis-tic-ex-pi-ali-do-cious");
|
||||
}
|
||||
|
||||
// Testcases adapted from tests included with libhyphen.
|
||||
// (Using only the UTF-8 dictionaries/tests, and omitting those that require
|
||||
// the extended hyphenation algorithm.)
|
||||
|
||||
#[test]
|
||||
fn base() {
|
||||
let dic_path = "tests/base.hyf";
|
||||
let dic = match unsafe { mapped_hyph::load_file(dic_path) } {
|
||||
Some(dic) => dic,
|
||||
_ => panic!("failed to load dictionary {}", dic_path),
|
||||
};
|
||||
let hyph = Hyphenator::new(&*dic);
|
||||
use std::fs::File;
|
||||
use std::io::{BufRead,BufReader};
|
||||
let words: Vec<String> = {
|
||||
let file = File::open("tests/base.word").unwrap();
|
||||
BufReader::new(file).lines().map(|l| l.unwrap()).collect()
|
||||
};
|
||||
let hyphs: Vec<String> = {
|
||||
let file = File::open("tests/base.hyph").unwrap();
|
||||
BufReader::new(file).lines().map(|l| l.unwrap()).collect()
|
||||
};
|
||||
for i in 0 .. words.len() {
|
||||
assert_eq!(hyph.hyphenate_word(&words[i], '='), hyphs[i]);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn compound() {
|
||||
let dic_path = "tests/compound.hyf";
|
||||
let dic = match unsafe { mapped_hyph::load_file(dic_path) } {
|
||||
Some(dic) => dic,
|
||||
_ => panic!("failed to load dictionary {}", dic_path),
|
||||
};
|
||||
let hyph = Hyphenator::new(&*dic);
|
||||
assert_eq!(hyph.hyphenate_word("motorcycle", '-'), "mo-tor-cy-cle");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn compound4() {
|
||||
let dic_path = "tests/compound4.hyf";
|
||||
let dic = match unsafe { mapped_hyph::load_file(dic_path) } {
|
||||
Some(dic) => dic,
|
||||
_ => panic!("failed to load dictionary {}", dic_path),
|
||||
};
|
||||
let hyph = Hyphenator::new(&*dic);
|
||||
assert_eq!(hyph.hyphenate_word("motorcycle", '-'), "motor-cycle");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn compound5() {
|
||||
let dic_path = "tests/compound5.hyf";
|
||||
let dic = match unsafe { mapped_hyph::load_file(dic_path) } {
|
||||
Some(dic) => dic,
|
||||
_ => panic!("failed to load dictionary {}", dic_path),
|
||||
};
|
||||
let hyph = Hyphenator::new(&*dic);
|
||||
assert_eq!(hyph.hyphenate_word("postea", '-'), "post-e-a");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn compound6() {
|
||||
let dic_path = "tests/compound6.hyf";
|
||||
let dic = match unsafe { mapped_hyph::load_file(dic_path) } {
|
||||
Some(dic) => dic,
|
||||
_ => panic!("failed to load dictionary {}", dic_path),
|
||||
};
|
||||
let hyph = Hyphenator::new(&*dic);
|
||||
assert_eq!(hyph.hyphenate_word("meaque", '-'), "me-a-que");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn settings2() {
|
||||
let dic_path = "tests/settings2.hyf";
|
||||
let dic = match unsafe { mapped_hyph::load_file(dic_path) } {
|
||||
Some(dic) => dic,
|
||||
_ => panic!("failed to load dictionary {}", dic_path),
|
||||
};
|
||||
let hyph = Hyphenator::new(&*dic);
|
||||
assert_eq!(hyph.hyphenate_word("őőőőőőő", '='), "ő=ő=ő=ő=ő=ő=ő");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn settings3() {
|
||||
let dic_path = "tests/settings3.hyf";
|
||||
let dic = match unsafe { mapped_hyph::load_file(dic_path) } {
|
||||
Some(dic) => dic,
|
||||
_ => panic!("failed to load dictionary {}", dic_path),
|
||||
};
|
||||
let hyph = Hyphenator::new(&*dic);
|
||||
assert_eq!(hyph.hyphenate_word("őőőőőőő", '='), "őő=ő=ő=ő=őő");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn hyphen() {
|
||||
let dic_path = "tests/hyphen.hyf";
|
||||
let dic = match unsafe { mapped_hyph::load_file(dic_path) } {
|
||||
Some(dic) => dic,
|
||||
_ => panic!("failed to load dictionary {}", dic_path),
|
||||
};
|
||||
let hyph = Hyphenator::new(&*dic);
|
||||
assert_eq!(hyph.hyphenate_word("foobar'foobar-foobar’foobar", '='), "foobar'foobar-foobar’foobar");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn lhmin() {
|
||||
let dic_path = "tests/lhmin.hyf";
|
||||
let dic = match unsafe { mapped_hyph::load_file(dic_path) } {
|
||||
Some(dic) => dic,
|
||||
_ => panic!("failed to load dictionary {}", dic_path),
|
||||
};
|
||||
let hyph = Hyphenator::new(&*dic);
|
||||
assert_eq!(hyph.hyphenate_word("miért", '='), "mi=ért");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rhmin() {
|
||||
let dic_path = "tests/rhmin.hyf";
|
||||
let dic = match unsafe { mapped_hyph::load_file(dic_path) } {
|
||||
Some(dic) => dic,
|
||||
_ => panic!("failed to load dictionary {}", dic_path),
|
||||
};
|
||||
let hyph = Hyphenator::new(&*dic);
|
||||
assert_eq!(hyph.hyphenate_word("övéit", '='), "övéit");
|
||||
assert_eq!(hyph.hyphenate_word("అంగడిధర", '='), "అం=గ=డిధర");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn num() {
|
||||
let dic_path = "tests/num.hyf";
|
||||
let dic = match unsafe { mapped_hyph::load_file(dic_path) } {
|
||||
Some(dic) => dic,
|
||||
_ => panic!("failed to load dictionary {}", dic_path),
|
||||
};
|
||||
let hyph = Hyphenator::new(&*dic);
|
||||
assert_eq!(hyph.hyphenate_word("foobar", '='), "foobar");
|
||||
assert_eq!(hyph.hyphenate_word("foobarfoobar", '='), "foobar=foobar");
|
||||
assert_eq!(hyph.hyphenate_word("barfoobarfoo", '='), "barfoo=barfoo");
|
||||
assert_eq!(hyph.hyphenate_word("123foobarfoobar", '='), "123foobar=foobar");
|
||||
assert_eq!(hyph.hyphenate_word("foobarfoobar123", '='), "foobar=foobar123");
|
||||
assert_eq!(hyph.hyphenate_word("123foobarfoobar123", '='), "123foobar=foobar123");
|
||||
assert_eq!(hyph.hyphenate_word("123barfoobarfoo", '='), "123barfoo=barfoo");
|
||||
assert_eq!(hyph.hyphenate_word("barfoobarfoo123", '='), "barfoo=barfoo123");
|
||||
assert_eq!(hyph.hyphenate_word("123barfoobarfoo123", '='), "123barfoo=barfoo123");
|
||||
}
|
||||
}
|
||||
|
|
@ -43,6 +43,7 @@ shift_or_euc_c = "0.1.0"
|
|||
audio_thread_priority = "0.20.2"
|
||||
mdns_service = { path="../../../../media/mtransport/mdns_service", optional = true }
|
||||
neqo_glue = { path = "../../../../netwerk/socket/neqo_glue" }
|
||||
mapped_hyph = { git = "https://github.com/jfkthame/mapped_hyph.git", branch = "master" }
|
||||
|
||||
[build-dependencies]
|
||||
rustc_version = "0.2"
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@
|
|||
|
||||
extern crate geckoservo;
|
||||
|
||||
extern crate mapped_hyph;
|
||||
extern crate kvstore;
|
||||
extern crate mp4parse_capi;
|
||||
extern crate nsstring;
|
||||
|
|
|
|||
Загрузка…
Ссылка в новой задаче