Merge pull request #65 from google/sr

Set normalization_rule in once place

src/sentencepiece_trainer.cc

@@ -32,7 +32,6 @@ static constexpr char kDefaultNormalizerName[] = "nfkc";
 // static
 util::Status SentencePieceTrainer::Train(const TrainerSpec &trainer_spec) {
   NormalizerSpec normalizer_spec;
-  normalizer_spec.set_name(kDefaultNormalizerName);
   Train(trainer_spec, normalizer_spec);
   return util::OkStatus();
 }
@@ -192,10 +191,7 @@ util::Status SentencePieceTrainer::MergeSpecsFromArgs(
 util::Status SentencePieceTrainer::Train(const std::string &args) {
   TrainerSpec trainer_spec;
   NormalizerSpec normalizer_spec;
-  normalizer_spec.set_name(kDefaultNormalizerName);
-
   CHECK_OK(MergeSpecsFromArgs(args, &trainer_spec, &normalizer_spec));
-
   return Train(trainer_spec, normalizer_spec);
 }
 

src/spm_train_main.cc

@@ -12,14 +12,14 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.!
 
-#include "builder.h"
+#include <map>
 #include "flags.h"
+#include "sentencepiece_model.pb.h"
 #include "sentencepiece_trainer.h"
 #include "util.h"
 
 using sentencepiece::NormalizerSpec;
 using sentencepiece::TrainerSpec;
-using sentencepiece::normalizer::Builder;
 
 namespace {
 static sentencepiece::TrainerSpec kDefaultTrainerSpec;
@@ -141,8 +141,8 @@ int main(int argc, char *argv[]) {
       {"word", TrainerSpec::WORD},
       {"char", TrainerSpec::CHAR}};
 
-  trainer_spec.set_model_type(
-      sentencepiece::port::FindOrDie(kModelTypeMap, FLAGS_model_type));
+  trainer_spec.set_model_type(sentencepiece::port::FindOrDie(
+      kModelTypeMap, sentencepiece::string_util::ToLower(FLAGS_model_type)));
 
   CHECK_OK(sentencepiece::SentencePieceTrainer::Train(trainer_spec,
                                                       normalizer_spec));