Fixed critical memory leak bug
This commit is contained in:
Родитель
032ae48af3
Коммит
c7cf151f2a
|
@ -158,7 +158,7 @@ We have evaluated SentencePiece segmentation with the following configurations.
|
||||||
|
|
||||||
* Segmentation algorithms:
|
* Segmentation algorithms:
|
||||||
* **BPE** (Byte Pair
|
* **BPE** (Byte Pair
|
||||||
Encoding) [[Sennrich et al.](http://www.aclweb.org/anthology/P16-1162)]] (`--model_type=bpe`)
|
Encoding) [[Sennrich et al.](http://www.aclweb.org/anthology/P16-1162)] (`--model_type=bpe`)
|
||||||
* **Unigram**. Language-model based segmentation. (`--model_type=unigram`)
|
* **Unigram**. Language-model based segmentation. (`--model_type=unigram`)
|
||||||
|
|
||||||
* Pre-tokenization methods:
|
* Pre-tokenization methods:
|
||||||
|
|
|
@ -68,8 +68,8 @@ void Lattice::Clear() {
|
||||||
end_nodes_.clear();
|
end_nodes_.clear();
|
||||||
sentence_.clear();
|
sentence_.clear();
|
||||||
surface_.clear();
|
surface_.clear();
|
||||||
all_nodes_.clear();
|
|
||||||
port::STLDeleteElements(&all_nodes_);
|
port::STLDeleteElements(&all_nodes_);
|
||||||
|
all_nodes_.clear();
|
||||||
}
|
}
|
||||||
|
|
||||||
void Lattice::SetSentence(StringPiece sentence) {
|
void Lattice::SetSentence(StringPiece sentence) {
|
||||||
|
@ -392,9 +392,8 @@ void ModelBase::BuildTrie(std::vector<std::pair<std::string, int>> *pieces) {
|
||||||
}
|
}
|
||||||
|
|
||||||
trie_ = port::MakeUnique<Darts::DoubleArray>();
|
trie_ = port::MakeUnique<Darts::DoubleArray>();
|
||||||
CHECK_EQ(0,
|
CHECK_EQ(0, trie_->build(key.size(), const_cast<char **>(&key[0]), nullptr,
|
||||||
trie_->build(key.size(), const_cast<char **>(&key[0]), nullptr,
|
&value[0]))
|
||||||
&value[0]))
|
|
||||||
<< "cannot build double-array";
|
<< "cannot build double-array";
|
||||||
|
|
||||||
// Computes the maximum number of shared prefixes in the trie.
|
// Computes the maximum number of shared prefixes in the trie.
|
||||||
|
|
Загрузка…
Ссылка в новой задаче