adding aurora4 eg

git-svn-id: https://svn.code.sf.net/p/kaldi/code/trunk@3292 5e6a8d80-dfce-4ca6-a32a-6e07a63d50c8
2013-12-10 22:24:27 +00:00 · 2013-12-10 22:24:27 +00:00 · 6a4251460f
--- a/egs/aurora4/s5/cmd.sh
+++ b/egs/aurora4/s5/cmd.sh
@ -0,0 +1,29 @@
+# "queue.pl" uses qsub.  The options to it are
+# options to qsub.  If you have GridEngine installed,
+# change this to a queue you have access to.
+# Otherwise, use "run.pl", which will run jobs locally
+# (make sure your --num-jobs options are no more than
+# the number of cpus on your machine.
+
+#a) JHU cluster options
+export train_cmd="queue.pl -l arch=*64"
+export decode_cmd="queue.pl -l arch=*64,mem_free=2G,ram_free=2G"
+export mkgraph_cmd="queue.pl -l arch=*64,ram_free=4G,mem_free=4G"
+
+#export cuda_cmd="..."
+
+
+#b) BUT cluster options
+#export train_cmd="queue.pl -q all.q@@blade -l ram_free=1200M,mem_free=1200M"
+#export decode_cmd="queue.pl -q all.q@@blade -l ram_free=1700M,mem_free=1700M"
+#export decodebig_cmd="queue.pl -q all.q@@blade -l ram_free=4G,mem_free=4G"
+
+#export cuda_cmd="queue.pl -q long.q@@pco203 -l gpu=1"
+#export cuda_cmd="queue.pl -q long.q@pcspeech-gpu"
+#export mkgraph_cmd="queue.pl -q all.q@@servers -l ram_free=4G,mem_free=4G"
+
+#c) run it locally...
+#export train_cmd=run.pl
+#export decode_cmd=run.pl
+export cuda_cmd=run.pl
+#export mkgraph_cmd=run.pl
--- a/egs/aurora4/s5/conf/decode_dnn.config
+++ b/egs/aurora4/s5/conf/decode_dnn.config
@ -0,0 +1,2 @@
+beam=18.0 # beam for decoding.  Was 13.0 in the scripts.
+latbeam=10.0 # this has most effect on size of the lattices.
--- a/egs/aurora4/s5/conf/fbank.conf
+++ b/egs/aurora4/s5/conf/fbank.conf
@ -0,0 +1,11 @@
+# No non-default options for now.
+--window-type=hamming # disable Dans window, use the standard
+--use-energy=false    # only fbank outputs
+--sample-frequency=16000 # Cantonese is sampled at 8kHz
+
+--low-freq=64         # typical setup from Frantisek Grezl
+--high-freq=8000
+--dither=1
+
+--num-mel-bins=40     # 8kHz so we use 15 bins
+--htk-compat=true     # try to make it compatible with HTK
--- a/egs/aurora4/s5/conf/mfcc.conf
+++ b/egs/aurora4/s5/conf/mfcc.conf
@ -0,0 +1 @@
+--use-energy=false   # only non-default option.
--- a/egs/aurora4/s5/conf/train_dnn.config
+++ b/egs/aurora4/s5/conf/train_dnn.config
@ -0,0 +1,9 @@
+momentum=0
+l1_penalty=0
+l2_penalty=0
+max_iters=25
+min_iters=
+start_halving_inc=0.5
+end_halving_inc=0.1
+halving_factor=0.7
+cache_size=32768
--- a/egs/aurora4/s5/conf/train_dnn_1.config
+++ b/egs/aurora4/s5/conf/train_dnn_1.config
@ -0,0 +1,9 @@
+momentum=0
+l1_penalty=0
+l2_penalty=0
+max_iters=30
+min_iters=
+start_halving_inc=0.5
+end_halving_inc=0.1
+halving_factor=0.5
+cache_size=65536
--- a/egs/aurora4/s5/conf/train_dnn_2.config
+++ b/egs/aurora4/s5/conf/train_dnn_2.config
@ -0,0 +1,9 @@
+momentum=0.9
+l1_penalty=0
+l2_penalty=0
+max_iters=10
+min_iters=10
+start_halving_inc=0.5
+end_halving_inc=0.1
+halving_factor=0.7
+cache_size=32768
--- a/egs/aurora4/s5/conf/train_nnet.config
+++ b/egs/aurora4/s5/conf/train_nnet.config
@ -0,0 +1,9 @@
+momentum=0
+l1_penalty=0
+l2_penalty=0
+max_iters=25
+min_iters=
+start_halving_inc=0.5
+end_halving_inc=0.1
+halving_factor=0.7
+cache_size=32768
--- a/egs/aurora4/s5/local/aurora2flist.pl
+++ b/egs/aurora4/s5/local/aurora2flist.pl
@ -0,0 +1,55 @@
+#!/usr/bin/perl
+
+# Copyright 2010-2011 Microsoft Corporation
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#  http://www.apache.org/licenses/LICENSE-2.0
+#
+# THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED
+# WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
+# MERCHANTABLITY OR NON-INFRINGEMENT.
+# See the Apache 2 License for the specific language governing permissions and
+# limitations under the License.
+
+# This is modified from the script in standard Kaldi recipe to account
+# for the way the WSJ data is structured on the Edinburgh systems. 
+# - Arnab Ghoshal, 12/1/12
+
+# This program takes as its standard input an .ndx file from the WSJ corpus that looks
+# like this:
+#;; File: tr_s_wv1.ndx, updated 04/26/94
+#;;
+#;; Index for WSJ0 SI-short Sennheiser training data
+#;; Data is read WSJ sentences, Sennheiser mic.
+#;; Contains 84 speakers X (~100 utts per speaker MIT/SRI and ~50 utts 
+#;; per speaker TI) = 7236 utts
+#;;
+#11_1_1:wsj0/si_tr_s/01i/01ic0201.wv1
+#11_1_1:wsj0/si_tr_s/01i/01ic0202.wv1
+#11_1_1:wsj0/si_tr_s/01i/01ic0203.wv1
+
+# and as command-line argument it takes the names of the WSJ disk locations, e.g.:
+# /group/corpora/public/wsjcam0/data on DICE machines.
+# It outputs a list of absolute pathnames.
+
+$wsj_dir = $ARGV[0];
+
+while(<STDIN>){
+  if(m/^;/){ next; } # Comment.  Ignore it.
+  else {
+    #m/^([0-9_]+):\s*(\S+)$/  || die "Could not parse line $_";
+    @A=split(" ", $_);
+    $filename = $A[0]; # as a subdirectory of the distributed disk.
+    #if ($filename !~ m/\.wv1$/) { $filename .= ".wv1"; }
+    $filename = "$wsj_dir/$filename";
+    if (-e $filename) {
+      print "$filename\n";
+    } else {
+      print STDERR "File $filename found in the index but not on disk\n";
+    }
+  }
+}
--- a/egs/aurora4/s5/local/aurora4_data_prep.sh
+++ b/egs/aurora4/s5/local/aurora4_data_prep.sh
@ -0,0 +1,401 @@
+#!/bin/bash
+set -e
+
+# Copyright 2009-2012  Microsoft Corporation  Johns Hopkins University (Author: Daniel Povey)
+# Apache 2.0.
+
+# This is modified from the script in standard Kaldi recipe to account
+# for the way the WSJ data is structured on the Edinburgh systems. 
+# - Arnab Ghoshal, 29/05/12
+
+if [ $# -ne 2 ]; then
+  printf "\nUSAGE: %s <corpus-directory>\n\n" `basename $0`
+  echo "The argument should be a the top-level WSJ corpus directory."
+  echo "It is assumed that there will be a 'wsj0' and a 'wsj1' subdirectory"
+  echo "within the top-level corpus directory."
+  exit 1;
+fi
+
+AURORA=$1
+CORPUS=$2
+
+dir=`pwd`/data/local/data
+lmdir=`pwd`/data/local/nist_lm
+mkdir -p $dir $lmdir
+local=`pwd`/local
+utils=`pwd`/utils
+
+. ./path.sh # Needed for KALDI_ROOT
+export PATH=$PATH:$KALDI_ROOT/tools/irstlm/bin
+sph2pipe=$KALDI_ROOT/tools/sph2pipe_v2.5/sph2pipe
+if [ ! -x $sph2pipe ]; then
+  echo "Could not find (or execute) the sph2pipe program at $sph2pipe";
+  exit 1;
+fi
+
+cd $dir
+
+# SI-84 clean training data
+cat $AURORA/lists/training_clean_sennh_16k.list \
+  | $local/aurora2flist.pl $AURORA | sort -u > train_si84_clean.flist
+
+# SI-84 multi-condition training data
+cat $AURORA/lists/training_multicondition_16k.list \
+  | $local/aurora2flist.pl $AURORA | sort -u > train_si84_multi.flist
+
+#Dev Set
+for x in $(seq -f "%02g" 01 14); do
+  # Dev-set 1 (330x14 utterances)
+  cat $AURORA/lists/devtest${x}_0330_16k.list | perl -e ' 
+    while(<STDIN>) {
+      @A=split("/", $_);        
+      @B=split("_", $A[0]);
+      print $B[0].$B[1]."_".$B[2]."/".$_;  
+    }
+  ' | $local/aurora2flist.pl $AURORA | sort -u > dev_0330_${x}.flist
+  # Dev-set 2 (1206x14 utterances)
+  cat $AURORA/lists/devtest${x}_1206_16k.list | perl -e '
+    while(<STDIN>) {
+      @A=split("/", $_);        
+      @B=split("_", $A[0]);
+      print $B[0].$B[1]."_".$B[2]."/".$_;  
+    }
+  ' | $local/aurora2flist.pl $AURORA | sort -u > dev_1206_${x}.flist
+done
+
+#Test Set
+for x in $(seq -f "%02g" 01 14); do
+  # test set 1 (166x14 utterances)
+  cat $AURORA/lists/test${x}_0166_16k.list \
+  | $local/aurora2flist.pl $AURORA | sort -u > test_0166_${x}.flist 
+  cat $AURORA/lists/test${x}_0330_16k.list \
+  | $local/aurora2flist.pl $AURORA | sort -u > test_eval92_${x}.flist
+done
+
+# Finding the transcript files:
+find -L $CORPUS -iname '*.dot' > dot_files.flist
+
+# Convert the transcripts into our format (no normalization yet)
+# adding suffix to utt_id 
+# 0 for clean condition
+
+# Trans and sph for Train Set
+x=train_si84_clean
+$local/flist2scp_12.pl $x.flist | sort > ${x}_sph_tmp.scp
+cat ${x}_sph_tmp.scp | awk '{print $1}' \
+  | $local/find_transcripts.pl dot_files.flist > ${x}_tmp.trans1
+cat ${x}_sph_tmp.scp | awk '{printf("%s0 %s\n", $1, $2);}' > ${x}_sph.scp
+cat ${x}_tmp.trans1 | awk '{printf("%s0 ", $1); for(i=2;i<=NF;i++) printf("%s ", $i); printf("\n");}' > ${x}.trans1
+
+x=train_si84_multi
+$local/flist2scp_12.pl $x.flist | sort > ${x}_sph_tmp.scp
+cat ${x}_sph_tmp.scp | awk '{print $1}' \
+  | $local/find_transcripts.pl dot_files.flist > ${x}_tmp.trans1
+cat ${x}_sph_tmp.scp | awk '{printf("%s1 %s\n", $1, $2);}' > ${x}_sph.scp
+cat ${x}_tmp.trans1 | awk '{printf("%s1 ", $1); for(i=2;i<=NF;i++) printf("%s ", $i); printf("\n");}' > ${x}.trans1
+
+# Trans and sph for Dev Set
+for x in $(seq -f "%02g" 01 14); do
+  $local/flist2scp_12.pl dev_0330_${x}.flist | sort > dev_0330_${x}_sph_tmp.scp
+  $local/flist2scp_12.pl dev_1206_${x}.flist | sort > dev_1206_${x}_sph_tmp.scp
+  cat dev_0330_${x}_sph_tmp.scp | awk '{print $1}' \
+    | $local/find_transcripts.pl dot_files.flist > dev_0330_${x}_tmp.trans1
+  cat dev_1206_${x}_sph_tmp.scp | awk '{print $1}' \
+    | $local/find_transcripts.pl dot_files.flist > dev_1206_${x}_tmp.trans1
+  cat dev_0330_${x}_sph_tmp.scp | perl -e \
+  ' $condition="$ARGV[0]";
+    if ($condition eq "01") {$suffix=0;}
+    elsif ($condition eq "02") {$suffix=1;} 
+    elsif ($condition eq "03") {$suffix=2;} 
+    elsif ($condition eq "04") {$suffix=3;} 
+    elsif ($condition eq "05") {$suffix=4;} 
+    elsif ($condition eq "06") {$suffix=5;} 
+    elsif ($condition eq "07") {$suffix=6;} 
+    elsif ($condition eq "08") {$suffix=7;} 
+    elsif ($condition eq "09") {$suffix=8;} 
+    elsif ($condition eq "10") {$suffix=9;} 
+    elsif ($condition eq "11") {$suffix=a;} 
+    elsif ($condition eq "12") {$suffix=b;} 
+    elsif ($condition eq "13") {$suffix=c;} 
+    elsif ($condition eq "14") {$suffix=d;} 
+    else {print STDERR "error condition $condition\n";}
+    while(<STDIN>) {
+      @A=split(" ", $_);  
+      print $A[0].$suffix." ".$A[1]."\n"; 
+    }
+  ' $x > dev_0330_${x}_sph.scp 
+  
+  cat dev_1206_${x}_sph_tmp.scp | perl -e \
+  ' $condition="$ARGV[0]";
+    if ($condition eq "01") {$suffix=0;}
+    elsif ($condition eq "02") {$suffix=1;} 
+    elsif ($condition eq "03") {$suffix=2;} 
+    elsif ($condition eq "04") {$suffix=3;} 
+    elsif ($condition eq "05") {$suffix=4;} 
+    elsif ($condition eq "06") {$suffix=5;} 
+    elsif ($condition eq "07") {$suffix=6;} 
+    elsif ($condition eq "08") {$suffix=7;} 
+    elsif ($condition eq "09") {$suffix=8;} 
+    elsif ($condition eq "10") {$suffix=9;} 
+    elsif ($condition eq "11") {$suffix=a;} 
+    elsif ($condition eq "12") {$suffix=b;} 
+    elsif ($condition eq "13") {$suffix=c;} 
+    elsif ($condition eq "14") {$suffix=d;} 
+    else {print STDERR "error condition $condition";}
+    while(<STDIN>) {
+      @A=split(" ", $_);  
+      print $A[0].$suffix." ".$A[1]."\n";
+    }
+  ' $x > dev_1206_${x}_sph.scp
+  
+  cat dev_0330_${x}_tmp.trans1 | perl -e \
+  ' $condition="$ARGV[0]";
+    if ($condition eq "01") {$suffix=0;}
+    elsif ($condition eq "02") {$suffix=1;} 
+    elsif ($condition eq "03") {$suffix=2;} 
+    elsif ($condition eq "04") {$suffix=3;} 
+    elsif ($condition eq "05") {$suffix=4;} 
+    elsif ($condition eq "06") {$suffix=5;} 
+    elsif ($condition eq "07") {$suffix=6;} 
+    elsif ($condition eq "08") {$suffix=7;} 
+    elsif ($condition eq "09") {$suffix=8;} 
+    elsif ($condition eq "10") {$suffix=9;} 
+    elsif ($condition eq "11") {$suffix=a;} 
+    elsif ($condition eq "12") {$suffix=b;} 
+    elsif ($condition eq "13") {$suffix=c;} 
+    elsif ($condition eq "14") {$suffix=d;} 
+    else {print STDERR "error condition $condition";}
+    while(<STDIN>) {
+      @A=split(" ", $_);  
+      print $A[0].$suffix;
+      for ($i=1; $i < @A; $i++) {print " ".$A[$i];}
+      print "\n";
+    }
+  ' $x > dev_0330_${x}.trans1
+ 
+  cat dev_1206_${x}_tmp.trans1 | perl -e \
+  ' $condition="$ARGV[0]";
+    if ($condition eq "01") {$suffix=0;}
+    elsif ($condition eq "02") {$suffix=1;} 
+    elsif ($condition eq "03") {$suffix=2;} 
+    elsif ($condition eq "04") {$suffix=3;} 
+    elsif ($condition eq "05") {$suffix=4;} 
+    elsif ($condition eq "06") {$suffix=5;} 
+    elsif ($condition eq "07") {$suffix=6;} 
+    elsif ($condition eq "08") {$suffix=7;} 
+    elsif ($condition eq "09") {$suffix=8;} 
+    elsif ($condition eq "10") {$suffix=9;} 
+    elsif ($condition eq "11") {$suffix=a;} 
+    elsif ($condition eq "12") {$suffix=b;} 
+    elsif ($condition eq "13") {$suffix=c;} 
+    elsif ($condition eq "14") {$suffix=d;} 
+    else {print STDERR "error condition $condition";}
+    while(<STDIN>) {
+      @A=split(" ", $_);  
+      print $A[0].$suffix;
+      for ($i=1; $i < @A; $i++) {print " ".$A[$i];}
+      print "\n";
+    }
+  ' $x > dev_1206_${x}.trans1
+
+done
+
+cat dev_0330_*_sph.scp | sort -k1 > dev_0330_sph.scp 
+cat dev_1206_*_sph.scp | sort -k1 > dev_1206_sph.scp
+cat dev_0330_??.trans1 | sort -k1 > dev_0330.trans1
+cat dev_1206_??.trans1 | sort -k1 > dev_1206.trans1
+
+
+# Trans and sph for Test Set
+for x in $(seq -f "%02g" 01 14); do
+  $local/flist2scp_12.pl test_0166_${x}.flist | sort > test_0166_${x}_sph_tmp.scp
+  $local/flist2scp_12.pl test_eval92_${x}.flist | sort > test_eval92_${x}_sph_tmp.scp
+  cat test_0166_${x}_sph_tmp.scp | awk '{print $1}' \
+    | $local/find_transcripts.pl dot_files.flist > test_0166_${x}_tmp.trans1
+  cat test_eval92_${x}_sph_tmp.scp | awk '{print $1}' \
+    | $local/find_transcripts.pl dot_files.flist > test_eval92_${x}_tmp.trans1
+  cat test_0166_${x}_sph_tmp.scp | perl -e \
+  ' $condition="$ARGV[0]";
+    if ($condition eq "01") {$suffix=0;}
+    elsif ($condition eq "02") {$suffix=1;} 
+    elsif ($condition eq "03") {$suffix=2;} 
+    elsif ($condition eq "04") {$suffix=3;} 
+    elsif ($condition eq "05") {$suffix=4;} 
+    elsif ($condition eq "06") {$suffix=5;} 
+    elsif ($condition eq "07") {$suffix=6;} 
+    elsif ($condition eq "08") {$suffix=7;} 
+    elsif ($condition eq "09") {$suffix=8;} 
+    elsif ($condition eq "10") {$suffix=9;} 
+    elsif ($condition eq "11") {$suffix=a;} 
+    elsif ($condition eq "12") {$suffix=b;} 
+    elsif ($condition eq "13") {$suffix=c;} 
+    elsif ($condition eq "14") {$suffix=d;} 
+    else {print STDERR "error condition $condition";}
+    while(<STDIN>) {
+      @A=split(" ", $_);  
+      print $A[0].$suffix." ".$A[1]."\n";
+    }
+  ' $x > test_0166_${x}_sph.scp 
+  
+  cat test_eval92_${x}_sph_tmp.scp | perl -e \
+  ' $condition="$ARGV[0]";
+    if ($condition eq "01") {$suffix=0;}
+    elsif ($condition eq "02") {$suffix=1;} 
+    elsif ($condition eq "03") {$suffix=2;} 
+    elsif ($condition eq "04") {$suffix=3;} 
+    elsif ($condition eq "05") {$suffix=4;} 
+    elsif ($condition eq "06") {$suffix=5;} 
+    elsif ($condition eq "07") {$suffix=6;} 
+    elsif ($condition eq "08") {$suffix=7;} 
+    elsif ($condition eq "09") {$suffix=8;} 
+    elsif ($condition eq "10") {$suffix=9;} 
+    elsif ($condition eq "11") {$suffix=a;} 
+    elsif ($condition eq "12") {$suffix=b;} 
+    elsif ($condition eq "13") {$suffix=c;} 
+    elsif ($condition eq "14") {$suffix=d;} 
+    else {print STDERR "error condition $condition";}
+    while(<STDIN>) {
+      @A=split(" ", $_);  
+      print $A[0].$suffix." ".$A[1]."\n";
+    }
+  ' $x > test_eval92_${x}_sph.scp
+  
+  cat test_0166_${x}_tmp.trans1 | perl -e \
+  ' $condition="$ARGV[0]";
+    if ($condition eq "01") {$suffix=0;}
+    elsif ($condition eq "02") {$suffix=1;} 
+    elsif ($condition eq "03") {$suffix=2;} 
+    elsif ($condition eq "04") {$suffix=3;} 
+    elsif ($condition eq "05") {$suffix=4;} 
+    elsif ($condition eq "06") {$suffix=5;} 
+    elsif ($condition eq "07") {$suffix=6;} 
+    elsif ($condition eq "08") {$suffix=7;} 
+    elsif ($condition eq "09") {$suffix=8;} 
+    elsif ($condition eq "10") {$suffix=9;} 
+    elsif ($condition eq "11") {$suffix=a;} 
+    elsif ($condition eq "12") {$suffix=b;} 
+    elsif ($condition eq "13") {$suffix=c;} 
+    elsif ($condition eq "14") {$suffix=d;} 
+    else {print STDERR "error condition $condition";}
+    while(<STDIN>) {
+      @A=split(" ", $_);  
+      print $A[0].$suffix;
+      for ($i=1; $i < @A; $i++) {print " ".$A[$i];}
+      print "\n";
+    }
+  ' $x > test_0166_${x}.trans1
+ 
+  cat test_eval92_${x}_tmp.trans1 | perl -e \
+  ' $condition="$ARGV[0]";
+    if ($condition eq "01") {$suffix=0;}
+    elsif ($condition eq "02") {$suffix=1;} 
+    elsif ($condition eq "03") {$suffix=2;} 
+    elsif ($condition eq "04") {$suffix=3;} 
+    elsif ($condition eq "05") {$suffix=4;} 
+    elsif ($condition eq "06") {$suffix=5;} 
+    elsif ($condition eq "07") {$suffix=6;} 
+    elsif ($condition eq "08") {$suffix=7;} 
+    elsif ($condition eq "09") {$suffix=8;} 
+    elsif ($condition eq "10") {$suffix=9;} 
+    elsif ($condition eq "11") {$suffix=a;} 
+    elsif ($condition eq "12") {$suffix=b;} 
+    elsif ($condition eq "13") {$suffix=c;} 
+    elsif ($condition eq "14") {$suffix=d;} 
+    else {print STDERR "error condition $condition";}
+    while(<STDIN>) {
+      @A=split(" ", $_);  
+      print $A[0].$suffix;
+      for ($i=1; $i < @A; $i++) {print " ".$A[$i];}
+      print "\n";
+    }
+  ' $x > test_eval92_${x}.trans1
+
+done
+
+cat test_0166_*_sph.scp | sort -k1 > test_0166_sph.scp 
+cat test_eval92_*_sph.scp | sort -k1 > test_eval92_sph.scp
+cat test_0166_??.trans1 | sort -k1 > test_0166.trans1
+cat test_eval92_??.trans1 | sort -k1 > test_eval92.trans1
+
+
+
+# Do some basic normalization steps.  At this point we don't remove OOVs--
+# that will be done inside the training scripts, as we'd like to make the
+# data-preparation stage independent of the specific lexicon used.
+noiseword="<NOISE>";
+for x in train_si84_clean train_si84_multi test_eval92 test_0166 dev_0330 dev_1206; do
+  cat $x.trans1 | $local/normalize_transcript.pl $noiseword \
+    | sort > $x.txt || exit 1;
+done
+
+# Create scp's with wav's. (the wv1 in the distribution is not really wav, it is sph.)
+for x in train_si84_clean train_si84_multi test_eval92 test_0166 dev_0330 dev_1206; do
+  awk '{printf("%s sox -B -r 16k -e signed -b 16 -c 1 -t raw %s -t wav - |\n", $1, $2);}' < ${x}_sph.scp \
+    > ${x}_wav.scp
+done
+
+# Make the utt2spk and spk2utt files.
+for x in train_si84_clean train_si84_multi test_eval92 test_0166 dev_0330 dev_1206; do
+  cat ${x}_sph.scp | awk '{print $1}' \
+    | perl -ane 'chop; m:^...:; print "$_ $&\n";' > $x.utt2spk
+  cat $x.utt2spk | $utils/utt2spk_to_spk2utt.pl > $x.spk2utt || exit 1;
+done
+
+#in case we want to limit lm's on most frequent words, copy lm training word frequency list
+cp $CORPUS/wsj0/doc/lng_modl/vocab/wfl_64.lst $lmdir
+chmod u+w $lmdir/*.lst # had weird permissions on source.
+
+# The 20K vocab, open-vocabulary language model (i.e. the one with UNK), without
+# verbalized pronunciations.   This is the most common test setup, I understand.
+
+cp $CORPUS/wsj0/doc/lng_modl/base_lm/bcb20onp.z $lmdir/lm_bg.arpa.gz || exit 1;
+chmod u+w $lmdir/lm_bg.arpa.gz
+
+# trigram would be:
+cat $CORPUS/wsj0/doc/lng_modl/base_lm/tcb20onp.z | \
+  perl -e 'while(<>){ if(m/^\\data\\/){ print; last;  } } while(<>){ print; }' \
+  | gzip -c -f > $lmdir/lm_tg.arpa.gz || exit 1;
+
+prune-lm --threshold=1e-7 $lmdir/lm_tg.arpa.gz $lmdir/lm_tgpr.arpa || exit 1;
+gzip -f $lmdir/lm_tgpr.arpa || exit 1;
+
+# repeat for 5k language models
+cp $CORPUS/wsj0/doc/lng_modl/base_lm/bcb05onp.z  $lmdir/lm_bg_5k.arpa.gz || exit 1;
+chmod u+w $lmdir/lm_bg_5k.arpa.gz
+
+# trigram would be: !only closed vocabulary here!
+cp $CORPUS/wsj0/doc/lng_modl/base_lm/tcb05cnp.z $lmdir/lm_tg_5k.arpa.gz || exit 1;
+chmod u+w $lmdir/lm_tg_5k.arpa.gz
+gunzip $lmdir/lm_tg_5k.arpa.gz
+tail -n 4328839 $lmdir/lm_tg_5k.arpa | gzip -c -f > $lmdir/lm_tg_5k.arpa.gz
+rm $lmdir/lm_tg_5k.arpa
+
+prune-lm --threshold=1e-7 $lmdir/lm_tg_5k.arpa.gz $lmdir/lm_tgpr_5k.arpa || exit 1;
+gzip -f $lmdir/lm_tgpr_5k.arpa || exit 1;
+
+
+if [ ! -f wsj0-train-spkrinfo.txt ] || [ `cat wsj0-train-spkrinfo.txt | wc -l` -ne 134 ]; then
+  rm -f wsj0-train-spkrinfo.txt
+  wget http://www.ldc.upenn.edu/Catalog/docs/LDC93S6A/wsj0-train-spkrinfo.txt \
+    || ( echo "Getting wsj0-train-spkrinfo.txt from backup location" && \
+         wget --no-check-certificate https://sourceforge.net/projects/kaldi/files/wsj0-train-spkrinfo.txt );
+fi
+
+if [ ! -f wsj0-train-spkrinfo.txt ]; then
+  echo "Could not get the spkrinfo.txt file from LDC website (moved)?"
+  echo "This is possibly omitted from the training disks; couldn't find it." 
+  echo "Everything else may have worked; we just may be missing gender info"
+  echo "which is only needed for VTLN-related diagnostics anyway."
+  exit 1
+fi
+# Note: wsj0-train-spkrinfo.txt doesn't seem to be on the disks but the
+# LDC put it on the web.  Perhaps it was accidentally omitted from the
+# disks.  
+
+cat $CORPUS/wsj0/doc/spkrinfo.txt \
+    ./wsj0-train-spkrinfo.txt  | \
+    perl -ane 'tr/A-Z/a-z/; m/^;/ || print;' | \
+    awk '{print $1, $2}' | grep -v -- -- | sort | uniq > spk2gender
+
+
+echo "Data preparation succeeded"
--- a/egs/aurora4/s5/local/aurora4_format_data.sh
+++ b/egs/aurora4/s5/local/aurora4_format_data.sh
@ -0,0 +1,86 @@
+#!/bin/bash
+
+# Copyright 2012  Microsoft Corporation  Johns Hopkins University (Author: Daniel Povey)
+# Apache 2.0
+
+# This script takes data prepared in a corpus-dependent way
+# in data/local/, and converts it into the "canonical" form,
+# in various subdirectories of data/, e.g. data/lang, data/lang_test_ug,
+# data/train_si284, data/train_si84, etc.
+
+# Don't bother doing train_si84 separately (although we have the file lists
+# in data/local/) because it's just the first 7138 utterances in train_si284.
+# We'll create train_si84 after doing the feature extraction.
+
+. ./path.sh || exit 1;
+
+echo "Preparing train and test data"
+srcdir=data/local/data
+lmdir=data/local/nist_lm
+tmpdir=data/local/lm_tmp
+lexicon=data/local/lang_tmp/lexiconp.txt
+mkdir -p $tmpdir
+
+for x in train_si84_clean train_si84_multi test_eval92 test_0166 dev_0330 dev_1206; do 
+  mkdir -p data/$x
+  cp $srcdir/${x}_wav.scp data/$x/wav.scp || exit 1;
+  cp $srcdir/$x.txt data/$x/text || exit 1;
+  cp $srcdir/$x.spk2utt data/$x/spk2utt || exit 1;
+  cp $srcdir/$x.utt2spk data/$x/utt2spk || exit 1;
+  utils/filter_scp.pl data/$x/spk2utt $srcdir/spk2gender > data/$x/spk2gender || exit 1;
+done
+
+
+# Next, for each type of language model, create the corresponding FST
+# and the corresponding lang_test_* directory.
+
+echo Preparing language models for test
+
+for lm_suffix in bg tgpr tg bg_5k tgpr_5k tg_5k; do
+  test=data/lang_test_${lm_suffix}
+  mkdir -p $test
+  for f in phones.txt words.txt phones.txt L.fst L_disambig.fst \
+     phones/; do
+    cp -r data/lang/$f $test
+  done
+  gunzip -c $lmdir/lm_${lm_suffix}.arpa.gz | \
+   utils/find_arpa_oovs.pl $test/words.txt  > $tmpdir/oovs_${lm_suffix}.txt
+
+  # grep -v '<s> <s>' because the LM seems to have some strange and useless
+  # stuff in it with multiple <s>'s in the history.  Encountered some other similar
+  # things in a LM from Geoff.  Removing all "illegal" combinations of <s> and </s>,
+  # which are supposed to occur only at being/end of utt.  These can cause 
+  # determinization failures of CLG [ends up being epsilon cycles].
+  gunzip -c $lmdir/lm_${lm_suffix}.arpa.gz | \
+    grep -v '<s> <s>' | \
+    grep -v '</s> <s>' | \
+    grep -v '</s> </s>' | \
+    arpa2fst - | fstprint | \
+    utils/remove_oovs.pl $tmpdir/oovs_${lm_suffix}.txt | \
+    utils/eps2disambig.pl | utils/s2eps.pl | fstcompile --isymbols=$test/words.txt \
+      --osymbols=$test/words.txt  --keep_isymbols=false --keep_osymbols=false | \
+     fstrmepsilon > $test/G.fst
+  fstisstochastic $test/G.fst
+ # The output is like:
+ # 9.14233e-05 -0.259833
+ # we do expect the first of these 2 numbers to be close to zero (the second is
+ # nonzero because the backoff weights make the states sum to >1).
+ # Because of the <s> fiasco for these particular LMs, the first number is not
+ # as close to zero as it could be.
+
+  # Everything below is only for diagnostic.
+  # Checking that G has no cycles with empty words on them (e.g. <s>, </s>);
+  # this might cause determinization failure of CLG.
+  # #0 is treated as an empty word.
+  mkdir -p $tmpdir/g
+  awk '{if(NF==1){ printf("0 0 %s %s\n", $1,$1); }} END{print "0 0 #0 #0"; print "0";}' \
+    < "$lexicon"  >$tmpdir/g/select_empty.fst.txt
+  fstcompile --isymbols=$test/words.txt --osymbols=$test/words.txt $tmpdir/g/select_empty.fst.txt | \
+   fstarcsort --sort_type=olabel | fstcompose - $test/G.fst > $tmpdir/g/empty_words.fst
+  fstinfo $tmpdir/g/empty_words.fst | grep cyclic | grep -w 'y' && 
+    echo "Language model has cycles with empty words" && exit 1
+  rm -r $tmpdir/g
+done
+
+echo "Succeeded in formatting data."
+rm -r $tmpdir
--- a/egs/aurora4/s5/local/cstr_ndx2flist.pl
+++ b/egs/aurora4/s5/local/cstr_ndx2flist.pl
@ -0,0 +1,54 @@
+#!/usr/bin/perl
+
+# Copyright 2010-2011 Microsoft Corporation
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#  http://www.apache.org/licenses/LICENSE-2.0
+#
+# THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED
+# WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
+# MERCHANTABLITY OR NON-INFRINGEMENT.
+# See the Apache 2 License for the specific language governing permissions and
+# limitations under the License.
+
+# This is modified from the script in standard Kaldi recipe to account
+# for the way the WSJ data is structured on the Edinburgh systems. 
+# - Arnab Ghoshal, 12/1/12
+
+# This program takes as its standard input an .ndx file from the WSJ corpus that looks
+# like this:
+#;; File: tr_s_wv1.ndx, updated 04/26/94
+#;;
+#;; Index for WSJ0 SI-short Sennheiser training data
+#;; Data is read WSJ sentences, Sennheiser mic.
+#;; Contains 84 speakers X (~100 utts per speaker MIT/SRI and ~50 utts 
+#;; per speaker TI) = 7236 utts
+#;;
+#11_1_1:wsj0/si_tr_s/01i/01ic0201.wv1
+#11_1_1:wsj0/si_tr_s/01i/01ic0202.wv1
+#11_1_1:wsj0/si_tr_s/01i/01ic0203.wv1
+
+# and as command-line argument it takes the names of the WSJ disk locations, e.g.:
+# /group/corpora/public/wsjcam0/data on DICE machines.
+# It outputs a list of absolute pathnames.
+
+$wsj_dir = $ARGV[0];
+
+while(<STDIN>){
+  if(m/^;/){ next; } # Comment.  Ignore it.
+  else {
+    m/^([0-9_]+):\s*(\S+)$/  || die "Could not parse line $_";
+    $filename = $2; # as a subdirectory of the distributed disk.
+    if ($filename !~ m/\.wv1$/) { $filename .= ".wv1"; }
+    $filename = "$wsj_dir/$filename";
+    if (-e $filename) {
+      print "$filename\n";
+    } else {
+      print STDERR "File $filename found in the index but not on disk\n";
+    }
+  }
+}
--- a/egs/aurora4/s5/local/cstr_wsj_data_prep.sh
+++ b/egs/aurora4/s5/local/cstr_wsj_data_prep.sh
@ -0,0 +1,187 @@
+#!/bin/bash
+set -e
+
+# Copyright 2009-2012  Microsoft Corporation  Johns Hopkins University (Author: Daniel Povey)
+# Apache 2.0.
+
+# This is modified from the script in standard Kaldi recipe to account
+# for the way the WSJ data is structured on the Edinburgh systems. 
+# - Arnab Ghoshal, 29/05/12
+
+if [ $# -ne 1 ]; then
+  printf "\nUSAGE: %s <corpus-directory>\n\n" `basename $0`
+  echo "The argument should be a the top-level WSJ corpus directory."
+  echo "It is assumed that there will be a 'wsj0' and a 'wsj1' subdirectory"
+  echo "within the top-level corpus directory."
+  exit 1;
+fi
+
+CORPUS=$1
+
+dir=`pwd`/data/local/data
+lmdir=`pwd`/data/local/nist_lm
+mkdir -p $dir $lmdir
+local=`pwd`/local
+utils=`pwd`/utils
+
+. ./path.sh # Needed for KALDI_ROOT
+export PATH=$PATH:$KALDI_ROOT/tools/irstlm/bin
+sph2pipe=$KALDI_ROOT/tools/sph2pipe_v2.5/sph2pipe
+if [ ! -x $sph2pipe ]; then
+  echo "Could not find (or execute) the sph2pipe program at $sph2pipe";
+  exit 1;
+fi
+
+cd $dir
+
+# This version for SI-84
+cat $CORPUS/wsj0/doc/indices/train/tr_s_wv1.ndx \
+  | $local/cstr_ndx2flist.pl $CORPUS | sort \
+  | grep -v wsj0/si_tr_s/401 > train_si84.flist
+
+# This version for SI-284
+cat $CORPUS/wsj1/doc/indices/si_tr_s.ndx \
+  $CORPUS/wsj0/doc/indices/train/tr_s_wv1.ndx \
+  | $local/cstr_ndx2flist.pl  $CORPUS | sort \
+  | grep -v wsj0/si_tr_s/401 > train_si284.flist
+
+# Now for the test sets.
+# $CORPUS/wsj1/doc/indices/readme.doc 
+# describes all the different test sets.
+# Note: each test-set seems to come in multiple versions depending
+# on different vocabulary sizes, verbalized vs. non-verbalized
+# pronunciations, etc.  We use the largest vocab and non-verbalized
+# pronunciations.
+# The most normal one seems to be the "baseline 60k test set", which
+# is h1_p0. 
+
+# Nov'92 (333 utts)
+# These index files have a slightly different format; 
+# have to add .wv1, which is done in cstr_ndx2flist.pl 
+cat $CORPUS/wsj0/doc/indices/test/nvp/si_et_20.ndx | \
+  $local/cstr_ndx2flist.pl $CORPUS | sort > test_eval92.flist
+
+# Nov'92 (330 utts, 5k vocab)
+cat $CORPUS/wsj0/doc/indices/test/nvp/si_et_05.ndx | \
+  $local/cstr_ndx2flist.pl $CORPUS | sort > test_eval92_5k.flist
+
+# Nov'93: (213 utts)
+# Have to replace a wrong disk-id.
+cat $CORPUS/wsj1/doc/indices/wsj1/eval/h1_p0.ndx | \
+  $local/cstr_ndx2flist.pl $CORPUS | sort > test_eval93.flist
+
+# Nov'93: (215 utts, 5k)
+cat $CORPUS/wsj1/doc/indices/wsj1/eval/h2_p0.ndx | \
+  $local/cstr_ndx2flist.pl $CORPUS | sort > test_eval93_5k.flist
+
+# Dev-set for Nov'93 (503 utts)
+cat $CORPUS/wsj1/doc/indices/h1_p0.ndx | \
+  $local/cstr_ndx2flist.pl $CORPUS | sort > test_dev93.flist
+
+# Dev-set for Nov'93 (513 utts, 5k vocab)
+cat $CORPUS/wsj1/doc/indices/h2_p0.ndx | \
+  $local/cstr_ndx2flist.pl $CORPUS | sort > test_dev93_5k.flist
+
+
+# Dev-set Hub 1,2 (503, 913 utterances)
+
+# Note: the ???'s below match WSJ and SI_DT, or wsj and si_dt.  
+# Sometimes this gets copied from the CD's with upcasing, don't know 
+# why (could be older versions of the disks).
+find $CORPUS/???1/??_??_20 -print | grep -i ".wv1" | sort > dev_dt_20.flist
+find $CORPUS/???1/??_??_05 -print | grep -i ".wv1" | sort > dev_dt_05.flist
+
+
+# Finding the transcript files:
+find -L $CORPUS -iname '*.dot' > dot_files.flist
+
+# Convert the transcripts into our format (no normalization yet)
+for x in train_si84 train_si284 test_eval92 test_eval93 test_dev93 test_eval92_5k test_eval93_5k test_dev93_5k dev_dt_05 dev_dt_20; do
+  $local/flist2scp.pl $x.flist | sort > ${x}_sph.scp
+  cat ${x}_sph.scp | awk '{print $1}' \
+    | $local/find_transcripts.pl dot_files.flist > $x.trans1
+done
+
+# Do some basic normalization steps.  At this point we don't remove OOVs--
+# that will be done inside the training scripts, as we'd like to make the
+# data-preparation stage independent of the specific lexicon used.
+noiseword="<NOISE>";
+for x in train_si84 train_si284 test_eval92 test_eval93 test_dev93 test_eval92_5k test_eval93_5k test_dev93_5k dev_dt_05 dev_dt_20; do
+  cat $x.trans1 | $local/normalize_transcript.pl $noiseword \
+    | sort > $x.txt || exit 1;
+done
+ 
+# Create scp's with wav's. (the wv1 in the distribution is not really wav, it is sph.)
+for x in train_si84 train_si284 test_eval92 test_eval93 test_dev93 test_eval92_5k test_eval93_5k test_dev93_5k dev_dt_05 dev_dt_20; do
+  awk '{printf("%s '$sph2pipe' -f wav %s |\n", $1, $2);}' < ${x}_sph.scp \
+    > ${x}_wav.scp
+done
+
+# Make the utt2spk and spk2utt files.
+for x in train_si84 train_si284 test_eval92 test_eval93 test_dev93 test_eval92_5k test_eval93_5k test_dev93_5k dev_dt_05 dev_dt_20; do
+  cat ${x}_sph.scp | awk '{print $1}' \
+    | perl -ane 'chop; m:^...:; print "$_ $&\n";' > $x.utt2spk
+  cat $x.utt2spk | $utils/utt2spk_to_spk2utt.pl > $x.spk2utt || exit 1;
+done
+
+#in case we want to limit lm's on most frequent words, copy lm training word frequency list
+cp $CORPUS/wsj1/doc/lng_modl/vocab/wfl_64.lst $lmdir
+chmod u+w $lmdir/*.lst # had weird permissions on source.
+
+# The 20K vocab, open-vocabulary language model (i.e. the one with UNK), without
+# verbalized pronunciations.   This is the most common test setup, I understand.
+
+cp $CORPUS/wsj1/doc/lng_modl/base_lm/bcb20onp.z $lmdir/lm_bg.arpa.gz || exit 1;
+chmod u+w $lmdir/lm_bg.arpa.gz
+
+# trigram would be:
+cat $CORPUS/wsj1/doc/lng_modl/base_lm/tcb20onp.z | \
+  perl -e 'while(<>){ if(m/^\\data\\/){ print; last;  } } while(<>){ print; }' \
+  | gzip -c -f > $lmdir/lm_tg.arpa.gz || exit 1;
+
+prune-lm --threshold=1e-7 $lmdir/lm_tg.arpa.gz $lmdir/lm_tgpr.arpa || exit 1;
+gzip -f $lmdir/lm_tgpr.arpa || exit 1;
+
+# repeat for 5k language models
+cp $CORPUS/wsj1/doc/lng_modl/base_lm/bcb05onp.z  $lmdir/lm_bg_5k.arpa.gz || exit 1;
+chmod u+w $lmdir/lm_bg_5k.arpa.gz
+
+# trigram would be: !only closed vocabulary here!
+cp $CORPUS/wsj1/doc/lng_modl/base_lm/tcb05cnp.z $lmdir/lm_tg_5k.arpa.gz || exit 1;
+chmod u+w $lmdir/lm_tg_5k.arpa.gz
+gunzip $lmdir/lm_tg_5k.arpa.gz
+tail -n 4328839 $lmdir/lm_tg_5k.arpa | gzip -c -f > $lmdir/lm_tg_5k.arpa.gz
+rm $lmdir/lm_tg_5k.arpa
+
+prune-lm --threshold=1e-7 $lmdir/lm_tg_5k.arpa.gz $lmdir/lm_tgpr_5k.arpa || exit 1;
+gzip -f $lmdir/lm_tgpr_5k.arpa || exit 1;
+
+
+if [ ! -f wsj0-train-spkrinfo.txt ] || [ `cat wsj0-train-spkrinfo.txt | wc -l` -ne 134 ]; then
+  rm -f wsj0-train-spkrinfo.txt
+  wget http://www.ldc.upenn.edu/Catalog/docs/LDC93S6A/wsj0-train-spkrinfo.txt \
+    || ( echo "Getting wsj0-train-spkrinfo.txt from backup location" && \
+         wget --no-check-certificate https://sourceforge.net/projects/kaldi/files/wsj0-train-spkrinfo.txt );
+fi
+
+if [ ! -f wsj0-train-spkrinfo.txt ]; then
+  echo "Could not get the spkrinfo.txt file from LDC website (moved)?"
+  echo "This is possibly omitted from the training disks; couldn't find it." 
+  echo "Everything else may have worked; we just may be missing gender info"
+  echo "which is only needed for VTLN-related diagnostics anyway."
+  exit 1
+fi
+# Note: wsj0-train-spkrinfo.txt doesn't seem to be on the disks but the
+# LDC put it on the web.  Perhaps it was accidentally omitted from the
+# disks.  
+
+cat $CORPUS/wsj0/doc/spkrinfo.txt \
+    $CORPUS/wsj1/doc/evl_spok/spkrinfo.txt \
+    $CORPUS/wsj1/doc/dev_spok/spkrinfo.txt \
+    $CORPUS/wsj1/doc/train/spkrinfo.txt \
+    ./wsj0-train-spkrinfo.txt  | \
+    perl -ane 'tr/A-Z/a-z/; m/^;/ || print;' | \
+    awk '{print $1, $2}' | grep -v -- -- | sort | uniq > spk2gender
+
+
+echo "Data preparation succeeded"
--- a/egs/aurora4/s5/local/cstr_wsj_extend_dict.sh
+++ b/egs/aurora4/s5/local/cstr_wsj_extend_dict.sh
@ -0,0 +1,172 @@
+#!/bin/bash
+
+# This script builds a larger word-list and dictionary 
+# than used for the LMs supplied with the WSJ corpus.
+# It uses a couple of strategies to fill-in words in
+# the LM training data but not in CMUdict.  One is
+# to generate special prons for possible acronyms, that
+# just consist of the constituent letters.  The other
+# is designed to handle derivatives of known words
+# (e.g. deriving the pron of a plural from the pron of
+# the base-word), but in a more general, learned-from-data
+# way.
+# It makes use of scripts in local/dict/
+
+if [ $# -ne 1 ]; then
+  echo "Usage: local/cstr_wsj_train_lms.sh WSJ1_doc_dir"
+  exit 1
+fi
+
+export PATH=$PATH:`pwd`/local/dict/
+srcdir=$1
+
+if [ ! -d $srcdir/lng_modl ]; then
+  echo "Expecting 'lng_modl' under WSJ doc directory '$srcdir'"
+  exit 1
+fi
+
+mkdir -p data/local/dict_larger
+dir=data/local/dict_larger
+cp data/local/dict/* data/local/dict_larger # Various files describing phones etc.
+  # are there; we just want to copy them as the phoneset is the same.
+rm data/local/dict_larger/lexicon.txt # we don't want this.
+mincount=2 # Minimum count of an OOV we will try to generate a pron for.
+
+[ ! -f data/local/dict/cmudict/cmudict.0.7a ] && echo "CMU dict not in expected place" && exit 1;
+
+# Remove comments from cmudict; print first field; remove
+# words like FOO(1) which are alternate prons: our dict format won't
+# include these markers.
+grep -v ';;;' data/local/dict/cmudict/cmudict.0.7a | 
+ perl -ane 's/^(\S+)\(\d+\)/$1/; print; ' | sort | uniq > $dir/dict.cmu
+
+cat $dir/dict.cmu | awk '{print $1}' | sort | uniq > $dir/wordlist.cmu
+
+echo "Getting training data [this should take at least a few seconds; if not, there's a problem]"
+
+# Convert to uppercase, remove XML-like markings.
+# For words ending in "." that are not in CMUdict, we assume that these
+# are periods that somehow remained in the data during data preparation,
+# and we we replace the "." with "\n".  Note: we found this by looking at
+# oov.counts below (before adding this rule).
+
+touch $dir/cleaned.gz
+if [ `du -m $dir/cleaned.gz | cut -f 1` -eq 73 ]; then
+  echo "Not getting cleaned data in $dir/cleaned.gz again [already exists]";
+else
+ gunzip -c $srcdir/lng_modl/lm_train/np_data/{87,88,89}/*.z \
+  | awk '/^</{next}{print toupper($0)}' | perl -e '
+   open(F, "<$ARGV[0]")||die;
+   while(<F>){ chop; $isword{$_} = 1; }
+   while(<STDIN>) { 
+    @A = split(" ", $_); 
+    for ($n = 0; $n < @A; $n++) {
+      $a = $A[$n];
+      if (! $isword{$a} && $a =~ s/^([^\.]+)\.$/$1/) { # nonwords that end in "."
+         # and have no other "." in them: treat as period.
+         print "$a";
+         if ($n+1 < @A) { print "\n"; }
+      } else { print "$a "; }
+    }
+    print "\n";
+  }
+ ' $dir/wordlist.cmu | gzip -c > $dir/cleaned.gz
+fi
+  
+# get unigram counts
+echo "Getting unigram counts"
+gunzip -c $dir/cleaned.gz | tr -s ' ' '\n' | \
+  awk '{count[$1]++} END{for (w in count) { print count[w], w; }}' | sort -nr > $dir/unigrams
+
+cat $dir/unigrams | awk -v dict=$dir/dict.cmu \
+  'BEGIN{while(getline<dict) seen[$1]=1;} {if(!seen[$2]){print;}}' \
+   > $dir/oov.counts
+
+echo "Most frequent unseen unigrams are: "
+head $dir/oov.counts
+
+# Prune away singleton counts, and remove things with numbers in
+# (which should have been normalized) and with no letters at all.
+
+
+cat $dir/oov.counts | awk -v thresh=$mincount '{if ($1 >= thresh) { print $2; }}' \
+  | awk '/[0-9]/{next;} /[A-Z]/{print;}' > $dir/oovlist
+
+# Automatic rule-finding...
+
+# First make some prons for possible acronyms.
+# Note: we don't do this for things like U.K or U.N,
+# or A.B. (which doesn't exist anyway), 
+# as we consider this normalization/spelling errors.
+
+cat $dir/oovlist | local/dict/get_acronym_prons.pl $dir/dict.cmu  > $dir/dict.acronyms
+
+mkdir $dir/f $dir/b # forward, backward directions of rules...
+  # forward is normal suffix
+  # rules, backward is reversed (prefix rules).  These
+  # dirs contain stuff we create while making the rule-based
+  # extensions to the dictionary.
+
+# Remove ; and , from words, if they are present; these
+# might crash our scripts, as they are used as separators there.
+filter_dict.pl $dir/dict.cmu > $dir/f/dict 
+cat $dir/oovlist | filter_dict.pl > $dir/f/oovs
+reverse_dict.pl $dir/f/dict > $dir/b/dict
+reverse_dict.pl $dir/f/oovs > $dir/b/oovs
+
+# The next stage takes a few minutes.
+# Note: the forward stage takes longer, as English is
+# mostly a suffix-based language, and there are more rules
+# that it finds.
+for d in $dir/f $dir/b; do
+ (
+   cd $d
+   cat dict | get_rules.pl 2>get_rules.log >rules
+   get_rule_hierarchy.pl rules >hierarchy
+   awk '{print $1}' dict | get_candidate_prons.pl rules dict | \
+     limit_candidate_prons.pl hierarchy | \
+     score_prons.pl dict | \
+     count_rules.pl >rule.counts
+   # the sort command below is just for convenience of reading.
+   score_rules.pl <rule.counts | sort -t';' -k3,3 -n -r >rules.with_scores
+   get_candidate_prons.pl rules.with_scores dict oovs | \
+     limit_candidate_prons.pl hierarchy > oovs.candidates
+ )  &   
+done 
+wait
+
+# Merge the candidates.
+reverse_candidates.pl $dir/b/oovs.candidates | cat - $dir/f/oovs.candidates | sort > $dir/oovs.candidates
+select_candidate_prons.pl <$dir/oovs.candidates | awk -F';' '{printf("%s  %s\n", $1, $2);}' \
+  > $dir/dict.oovs
+
+cat $dir/dict.acronyms $dir/dict.oovs | sort | uniq > $dir/dict.oovs_merged
+
+awk '{print $1}' $dir/dict.oovs_merged | uniq > $dir/oovlist.handled
+sort $dir/oovlist | diff - $dir/oovlist.handled  | grep -v 'd' | sed 's:< ::' > $dir/oovlist.not_handled
+
+
+# add_counts.pl attaches to original counts to the list of handled/not-handled OOVs
+add_counts.pl $dir/oov.counts $dir/oovlist.handled | sort -nr > $dir/oovlist.handled.counts
+add_counts.pl $dir/oov.counts $dir/oovlist.not_handled | sort -nr > $dir/oovlist.not_handled.counts
+
+echo "**Top OOVs we handled are:**"; 
+head $dir/oovlist.handled.counts
+echo "**Top OOVs we didn't handle are as follows (note: they are mostly misspellings):**"; 
+head $dir/oovlist.not_handled.counts
+
+
+echo "Count of OOVs we handled is `awk '{x+=$1} END{print x}' $dir/oovlist.handled.counts`"
+echo "Count of OOVs we couldn't handle is `awk '{x+=$1} END{print x}' $dir/oovlist.not_handled.counts`"
+echo "Count of OOVs we didn't handle due to low count is" \
+    `awk -v thresh=$mincount '{if ($1 < thresh) x+=$1; } END{print x;}' $dir/oov.counts`
+# The two files created above are for humans to look at, as diagnostics.
+
+cat <<EOF | cat - $dir/dict.cmu $dir/dict.oovs_merged | sort | uniq > $dir/lexicon.txt
+!SIL SIL
+<SPOKEN_NOISE> SPN
+<UNK> SPN
+<NOISE> NSN
+EOF
+
+echo "Created $dir/lexicon.txt"
--- a/egs/aurora4/s5/local/dict/add_counts.pl
+++ b/egs/aurora4/s5/local/dict/add_counts.pl
@ -0,0 +1,31 @@
+#!/usr/bin/perl
+
+
+# Add counts to an oovlist.
+# Reads in counts as output by uniq -c, and
+# an oovlist, and prints out the counts of the oovlist.
+
+(@ARGV == 1 || @ARGV == 2) || die "Usage: add_counts.pl count_file [oovlist]\n";
+
+$counts = shift @ARGV;
+
+open(C, "<$counts") || die "Opening counts file $counts";
+
+while(<C>) {
+  @A = split(" ", $_);
+  @A == 2 || die "Bad line in counts file: $_";
+  ($count, $word) = @A;
+  $count =~ m:^\d+$: || die "Bad count $A[0]\n";
+  $counts{$word} = $count;
+}
+
+while(<>) {
+  chop;
+  $w = $_;
+  $w =~ m:\S+: || die "Bad word $w";
+  defined $counts{$w} || die "Word $w not present in counts file";
+  print "\t$counts{$w}\t$w\n";
+}
+    
+  
+
--- a/egs/aurora4/s5/local/dict/count_rules.pl
+++ b/egs/aurora4/s5/local/dict/count_rules.pl
@ -0,0 +1,44 @@
+#!/usr/bin/perl
+
+# This program takes the output of score_prons.pl and collates
+# it for each (rule, destress) pair so that we get the
+# counts of right/partial/wrong for each pair.
+
+# The input is a 7-tuple on each line, like:
+# word;pron;base-word;base-pron;rule-name;de-stress;right|partial|wrong
+#
+# The output format is a 5-tuple like:
+#
+# rule;destress;right-count;partial-count;wrong-count
+#
+
+if (@ARGV != 0 && @ARGV != 1) {
+  die "Usage: count_rules.pl < scored_candidate_prons > rule_counts";
+}
+
+
+while(<>) {
+  chop;
+  $line = $_;
+  my ($word, $pron, $baseword, $basepron, $rulename, $destress, $score) = split(";", $line);
+  
+  my $key = $rulename . ";" . $destress;
+
+  if (!defined $counts{$key}) {
+    $counts{$key} = [ 0, 0, 0 ]; # new anonymous array.
+  }
+  $ref = $counts{$key};
+  if ($score eq "right") {
+    $$ref[0]++;
+  } elsif ($score eq "partial") {
+    $$ref[1]++;
+  } elsif ($score eq "wrong") {
+    $$ref[2]++;
+  } else {
+    die "Bad score $score\n";
+  }
+}
+
+while ( my ($key, $value) = each(%counts)) {
+  print $key . ";" . join(";", @$value) . "\n";
+}
--- a/egs/aurora4/s5/local/dict/filter_dict.pl
+++ b/egs/aurora4/s5/local/dict/filter_dict.pl
@ -0,0 +1,19 @@
+#!/usr/bin/perl
+
+
+# This program reads and writes either a dictionary or just a list
+# of words, and it removes any words containing ";" or "," as these
+# are used in these programs.  It will warn about these.
+# It will die if the pronunciations have these symbols in.
+while(<>) {
+  chop;
+  @A = split(" ", $_);
+  $word = shift @A;
+  
+  if ($word =~ m:[;,]:) {
+    print STDERR "Omitting line $_ since it has one of the banned characters ; or ,\n" ;
+  } else {
+    $_ =~ m:[;,]: && die "Phones cannot have ; or , in them.";
+    print $_ . "\n";
+  }
+}
--- a/egs/aurora4/s5/local/dict/find_acronyms.pl
+++ b/egs/aurora4/s5/local/dict/find_acronyms.pl
@ -0,0 +1,95 @@
+#!/usr/bin/perl
+
+# Reads a dictionary, and prints out a list of words that seem to be pronounced
+# as acronyms (not including plurals of acronyms, just acronyms).  Uses
+# the prons of the individual letters (A., B. and so on) to judge this.
+# Note: this is somewhat dependent on the convention used in CMUduct, that
+# the individual letters are spelled this way (e.g. "A.").
+
+$max_length = 6; # Max length of words that might be
+ # acronyms.
+
+while(<>) { # Read the dict.
+  chop;
+  @A = split(" ", $_);
+  $word = shift @A;
+  $pron = join(" ", @A);
+  if ($word =~ m/^([A-Z])\.$/ ) {
+    chop $word; # Remove trailing "." to get just the letter
+    $letter = $1;
+    if (!defined $letter_prons{$letter} ) { 
+      $letter_prons{$letter} = [ ]; # new anonymous array
+    }
+    $arrayref = $letter_prons{$letter};
+    push @$arrayref, $pron;
+  } elsif( length($word) <= $max_length ) {
+    $pronof{$word . "," . $pron} = 1;
+    $isword{$word} = 1;
+    #if (!defined $prons{$word} ) {
+    #  $prons{$word} = [ ];
+    #}
+    #  push @{$prons{$word}}, $pron;
+  }
+}
+
+sub get_letter_prons;
+
+foreach $word (keys %isword) {
+  my @letter_prons = get_letter_prons($word);
+  foreach $pron (@letter_prons) {
+    if (defined $pronof{$word.",".$pron}) {
+      print "$word  $pron\n";
+    }
+  }
+}
+
+
+sub get_letter_prons {
+  @acronym = split("", shift); # The letters in the word.
+  my @prons = ( "" );
+  
+  while (@acronym > 0) {
+    $l = shift @acronym;
+    $n = 1; # num-repeats of letter $l.
+    while (@acronym > 0 && $acronym[0] eq $l) {
+      $n++;
+      shift @acronym;
+    }
+    my $arrayref = $letter_prons{$l};
+    my @prons_of_block = ();
+    if ($n == 1) { # Just one repeat.
+      foreach $lpron ( @$arrayref ) {
+        push @prons_of_block, $lpron; # typically (always?) just one pron of a letter.
+      }
+    } elsif ($n == 2) { # Two repeats.  Can be "double a" or "a a"
+      foreach $lpron ( @$arrayref ) {
+        push @prons_of_block, "D AH1 B AH0 L " . $lpron;
+        push @prons_of_block, $lpron . $lpron;
+      }
+    } elsif ($n == 3) { # can be "triple a" or "a a a"
+      foreach $lpron ( @$arrayref ) {
+        push @prons_of_block, "T R IH1 P AH0 L " . $lpron;
+        push @prons_of_block, $lpron . $lpron . $lpron;
+      }
+    } elsif ($n >= 4) { # let's say it can only be that letter repeated $n times..
+      # not sure really.
+      foreach $lpron ( @$arrayref ) {
+        $nlpron = "";
+        for ($m = 0; $m < $n; $m++) { $nlpron = $nlpron . $lpron; }
+        push @prons_of_block, $nlpron;
+      }
+    }
+    my @new_prons = ();
+    foreach $pron (@prons) {
+      foreach $pron_of_block(@prons_of_block) {
+        if ($pron eq "") {
+          push @new_prons, $pron_of_block;
+        } else {
+          push @new_prons, $pron . " " . $pron_of_block;
+        }
+      }
+    }
+    @prons = @new_prons;
+  }
+  return @prons;
+}
--- a/egs/aurora4/s5/local/dict/get_acronym_prons.pl
+++ b/egs/aurora4/s5/local/dict/get_acronym_prons.pl
@ -0,0 +1,123 @@
+#!/usr/bin/perl
+
+# Reads a dictionary (for prons of letters), and an OOV list,
+# and puts out candidate pronunciations of words in that list
+# that could plausibly be acronyms.
+# We judge that a word can plausibly be an acronym if it is
+# a sequence of just letters (no non-letter characters such
+# as "'"),  or something like U.K.,
+# and the number of letters is four or less.
+#
+# If the text were not already pre-normalized, there would
+# be other hints such as capitalization.
+
+# This program appends
+# the prons of the individual letters (A., B. and so on) to work out
+# the pron of the acronym.
+# Note: this is somewhat dependent on the convention used in CMUduct, that
+# the individual letters are spelled this way (e.g. "A."). [it seems
+# to also have the separated versions.
+
+if (!(@ARGV == 1 || @ARGV == 2)) { 
+  print "Usage: get_acronym_prons.pl dict [oovlist]";
+}
+
+$max_length = 4; # Max #letters in an acronym. (Longer 
+ # acronyms tend to have "pseudo-pronunciations", e.g. think about UNICEF.
+
+$dict = shift @ARGV;
+open(D, "<$dict") || die "Opening dictionary $dict";
+
+while(<D>) { # Read the dict, to get the prons of the letters.
+  chop;
+  @A = split(" ", $_);
+  $word = shift @A;
+  $pron = join(" ", @A);
+  if ($word =~ m/^([A-Z])\.$/ ) {
+    chop $word; # Remove trailing "." to get just the letter
+    $letter = $1;
+    if (!defined $letter_prons{$letter} ) { 
+      $letter_prons{$letter} = [ ]; # new anonymous array
+    }
+    $arrayref = $letter_prons{$letter};
+    push @$arrayref, $pron;
+  } elsif( length($word) <= $max_length ) {
+    $pronof{$word . "," . $pron} = 1;
+    $isword{$word} = 1;
+    #if (!defined $prons{$word} ) {
+    #  $prons{$word} = [ ];
+    #}
+    #  push @{$prons{$word}}, $pron;
+  }
+}
+
+sub get_letter_prons;
+
+while(<>) { # Read OOVs.
+  # For now, just do the simple cases without "." in 
+  # between... things with "." in the OOV list seem to
+  # be mostly errors.
+  chop;
+  $word = $_;
+  if ($word =~ m/^[A-Z]{1,5}$/) {
+    foreach $pron ( get_letter_prons($word) ) { # E.g. UNPO
+      print "$word  $pron\n";
+    }
+  } elsif ($word =~ m:^(\w\.){1,4}\w\.?$:) { # E.g. U.K.  Make the final "." optional.
+    $letters = $word;
+    $letters =~ s:\.::g;
+    foreach $pron ( get_letter_prons($letters) ) { 
+      print "$word  $pron\n";
+    }
+  }
+}
+
+sub get_letter_prons {
+  @acronym = split("", shift); # The letters in the word.
+  my @prons = ( "" );
+  
+  while (@acronym > 0) {
+    $l = shift @acronym;
+    $n = 1; # num-repeats of letter $l.
+    while (@acronym > 0 && $acronym[0] eq $l) {
+      $n++;
+      shift @acronym;
+    }
+    my $arrayref = $letter_prons{$l};
+    my @prons_of_block = ();
+    if ($n == 1) { # Just one repeat.
+      foreach $lpron ( @$arrayref ) {
+        push @prons_of_block, $lpron; # typically (always?) just one pron of a letter.
+      }
+    } elsif ($n == 2) { # Two repeats.  Can be "double a" or "a a"
+      foreach $lpron ( @$arrayref ) {
+        push @prons_of_block, "D AH1 B AH0 L " . $lpron;
+        push @prons_of_block, $lpron . " " . $lpron;
+      }
+    } elsif ($n == 3) { # can be "triple a" or "a a a"
+      foreach $lpron ( @$arrayref ) {
+        push @prons_of_block, "T R IH1 P AH0 L " . $lpron;
+        push @prons_of_block, "$lpron $lpron $lpron";
+      }
+    } elsif ($n >= 4) { # let's say it can only be that letter repeated $n times..
+      # not sure really.
+      foreach $lpron ( @$arrayref ) {
+        $nlpron = $lpron;
+        for ($m = 1; $m < $n; $m++) { $nlpron = $nlpron . " " . $lpron; }
+        push @prons_of_block, $nlpron;
+      }
+    }
+    my @new_prons = ();
+    foreach $pron (@prons) {
+      foreach $pron_of_block(@prons_of_block) {
+        if ($pron eq "") {
+          push @new_prons, $pron_of_block;
+        } else {
+          push @new_prons, $pron . " " . $pron_of_block;
+        }
+      }
+    }
+    @prons = @new_prons;
+  }
+  return @prons;
+}
--- a/egs/aurora4/s5/local/dict/get_candidate_prons.pl
+++ b/egs/aurora4/s5/local/dict/get_candidate_prons.pl
@ -0,0 +1,187 @@
+#!/usr/bin/perl
+
+# This script takes three command-line arguments (typically files, or "-"):
+# the suffix rules (as output by get_rules.pl), the rule-hierarchy 
+# (from get_rule_hierarchy.pl), and the words that we want prons to be 
+# generated for (one per line).
+
+# The output consists of candidate generated pronunciations for those words,
+# together with information about how we generated those pronunciations.
+# This does not do pruning of the candidates using the restriction
+# "you can't use a more general rule when a more specific one is applicable".
+# That is done by limit_candidate_prons.pl.
+
+# Each line of the output consists of a 4-tuple, separated by ";", of the
+# form:
+# word;pron;base-word;base-pron;rule-name;destress[;rule-score]
+# [the last field is only present if you supplied rules with score information].
+# where:
+# - "word" is the input word that we queried for, e.g. WASTED
+# - "pron" is the generated pronunciation, e.g. "W EY1 S T AH0 D"
+# - rule-name is a 4-tuple separated by commas that describes the rule, e.g.
+#   "STED,STING,D,NG",
+# - "base-word" is the base-word we're getting the pron from,
+#   e.g. WASTING
+# - "base-pron" is the pron of the base-word, e.g. "W EY1 S T IH0 NG"
+# - "destress" is either "yes" or "no" and corresponds to whether we destressed the
+#   base-word or not [de-stressing just corresponds to just taking any 2's down to 1's,
+#   although we may extend this in future]... 
+# - "rule-score" is a numeric score of the rule (this field is only present
+#   if there was score information in your rules.
+
+
+(@ARGV == 2  || @ARGV == 3) || die "Usage: get_candidate_prons.pl rules base-dict [ words ]";
+
+$min_prefix_len = 3;  # this should probably match with get_rules.pl
+
+$rules = shift @ARGV; # Note: rules may be with destress "yes/no" indicators or without...
+                      # if without, it's treated as if both "yes" and "no" are present.
+$dict = shift @ARGV;
+
+open(R, "<$rules") || die "Opening rules file: $rules";
+
+sub process_word;
+
+while(<R>) {
+  chop $_;
+  my ($rule, $destress, $rule_score) = split(";", $_); # We may have "destress" markings (yes|no),
+  # and scores, or we may have just rule, in which case
+  # $destress and $rule_score will be undefined.
+
+  my @R = split(",", $rule, 4); # "my" means new instance of @R each
+  # time we do this loop -> important because we'll be creating
+  # a reference to @R below.
+  # Note: the last arg to SPLIT tells it how many fields max to get.
+  # This stops it from omitting empty trailing fields.
+  @R == 4 || die "Bad rule $_";
+  $suffix = $R[0]; # Suffix of word we want pron for.
+  if (!defined $isrule{$rule}) {
+    $isrule{$rule} = 1; # make sure we do this only once for each rule 
+    # (don't repeate for different stresses).
+    if (!defined $suffix2rule{$suffix}) {
+      # The syntax [ $x, $y, ... ] means a reference to a newly created array
+      # containing $x, $y, etc.   \@R creates an array reference to R.
+      # so suffix2rule is a hash from suffix to ref to array of refs to 
+      # 4-dimensional arrays.
+      $suffix2rule{$suffix} = [ \@R ];
+    } else {
+      # Below, the syntax @{$suffix2rule{$suffix}} dereferences the array
+      # reference inside the hash; \@R pushes onto that array a new array
+      # reference pointing to @R.
+      push @{$suffix2rule{$suffix}}, \@R;
+    }
+  }
+  if (!defined $rule_score) { $rule_score = -1; } # -1 means we don't have the score info.
+  
+  # Now store information on which destress markings (yes|no) this rule
+  # is valid for, and the associated scores (if supplied)
+  # If just the rule is given (i.e. no destress marking specified),
+  # assume valid for both.
+  if (!defined $destress) { # treat as if both "yes" and "no" are valid.
+    $rule_and_destress_to_rule_score{$rule.";yes"} = $rule_score;
+    $rule_and_destress_to_rule_score{$rule.";no"} = $rule_score;
+  } else {
+    $rule_and_destress_to_rule_score{$rule.";".$destress} = $rule_score;
+  }
+
+}
+
+open(D, "<$dict") || die "Opening base dictionary: $dict";
+while(<D>) {
+  @A = split(" ", $_);
+  $word = shift @A;
+  $pron = join(" ", @A);
+  if (!defined $word2prons{$word}) {
+    $word2prons{$word} = [ $pron ]; # Ref to new anonymous array containing just "pron".
+  } else {
+    push @{$word2prons{$word}}, $pron; # Push $pron onto array referred to (@$ref derefs array).
+  }
+}
+foreach $word (%word2prons) {
+  # Set up the hash "prefixcount", which says how many times a char-sequence
+  # is a prefix (not necessarily a strict prefix) of a word in the dict.
+  $len = length($word);
+  for ($l = 0; $l <= $len; $l++) {
+    $prefixcount{substr($word, 0, $l)}++;
+  }
+}
+
+open(R, "<$rules") || die "Opening rules file: $rules";
+
+
+while(<>) {
+  chop;
+  m/^\S+$/ || die;
+  process_word($_);
+}
+
+sub process_word {
+  my $word = shift @_;
+  $len = length($word);
+  # $owncount is used in evaluating whether a particular prefix is a prefix
+  # of some other word in the dict... if a word itself may be in the dict
+  # (usually because we're running this on the dict itself), we need to
+  # correct for this.
+  if (defined $word2prons{$word}) { $owncount = 1; } else { $owncount = 0; }
+  
+  for ($prefix_len = $min_prefix_len; $prefix_len <= $len; $prefix_len++) {
+    my $prefix = substr($word, 0, $prefix_len);
+    my $suffix = substr($word, $prefix_len);
+    if ($prefixcount{$prefix} - $owncount == 0) {
+      # This prefix is not a prefix of any word in the dict, so no point
+      # checking the rules below-- none of them can match.
+      next;
+    }
+    $rules_array_ref = $suffix2rule{$suffix};
+    if (defined $rules_array_ref) {
+      foreach $R (@$rules_array_ref) { # @$rules_array_ref dereferences the array.
+        # $R is a refernce to a 4-dimensional array, whose elements we access with
+        # $$R[0], etc.
+        my $base_suffix = $$R[1];
+        my $base_word = $prefix . $base_suffix;
+        my $base_prons_ref = $word2prons{$base_word};
+        if (defined $base_prons_ref) {
+          my $psuffix = $$R[2];
+          my $base_psuffix = $$R[3];
+          if ($base_psuffix ne "") { 
+            $base_psuffix = " " . $base_psuffix; 
+            # Include " ", the space between phones, to prevent
+            # matching partial phones below.
+          }
+          my $base_psuffix_len = length($base_psuffix);
+          foreach $base_pron (@$base_prons_ref) { # @$base_prons_ref derefs 
+            # that reference to an array.
+            my $base_pron_prefix_len = length($base_pron) - $base_psuffix_len;
+            # Note: these lengths are in characters, not phones.
+            if ($base_pron_prefix_len >= 0 && 
+                substr($base_pron, $base_pron_prefix_len) eq $base_psuffix) {
+              # The suffix of the base_pron is what it should be.
+              my $pron_prefix = substr($base_pron, 0, $base_pron_prefix_len);
+              my $rule = join(",", @$R); # we'll output this..
+              my $len = @R;
+              for ($destress = 0; $destress <= 1; $destress++) { # Two versions 
+                # of each rule: with destressing and without.
+                # pron is the generated pron.
+                if ($destress) {  $pron_prefix =~ s/2/1/g; }
+                my $pron;
+                if ($psuffix ne "") { $pron = $pron_prefix . " " . $psuffix; }
+                else { $pron = $pron_prefix; }
+                # Now print out the info about the generated pron.
+                my $destress_mark = ($destress ? "yes" : "no");
+                my $rule_score = $rule_and_destress_to_rule_score{$rule.";".$destress_mark};
+                if (defined $rule_score) { # Means that the (rule,destress) combination was
+                  # seen [note: this if-statement may be pointless, as currently we don't
+                  # do any pruning of rules].
+                  my @output = ($word, $pron, $base_word, $base_pron, $rule, $destress_mark);
+                  if ($rule_score != -1) { push @output, $rule_score; } # If scores were supplied,
+                  # we also output the score info.
+                  print join(";", @output) . "\n";
+                }
+              }
+            }  
+          }
+        }
+      }
+    }
+  }
+}  
--- a/egs/aurora4/s5/local/dict/get_rule_hierarchy.pl
+++ b/egs/aurora4/s5/local/dict/get_rule_hierarchy.pl
@ -0,0 +1,73 @@
+#!/usr/bin/perl
+
+#This reads in rules, of the form put out by get_rules.pl, e.g.:
+# ERT,,ER0 T,
+# MENT,ING,M AH0 N T,IH0 NG
+# S,TON,Z,T AH0 N
+# ,ER,IH0 NG,IH0 NG ER0
+# ,'S,M AH0 N,M AH0 N Z
+#TIONS,TIVE,SH AH0 N Z,T IH0 V
+
+# and it works out a hierarchy that says which rules are sub-cases
+# of which rules: it outputs on each line a pair separated by ";", where
+# each member of the pair is a rule, first one is the specialization, the
+# second one being more general.
+# E.g.:
+# RED,RE,D,/ED,E,D,
+# RED,RE,D,/D,,D,
+# GING,GE,IH0 NG,/ING,I,IH0 NG,
+# TOR,TING,T ER0,T IH0 NG/OR,OR,T ER0,T ER0 
+# ERED,ER,D,/RED,R,D,
+# ERED,ER,D,/ED,,D,
+
+
+
+
+while(<>) {
+  chop;
+  $rule = $_;
+  $isrule{$rule} = 1;
+  push @rules, $rule;
+}
+
+foreach my $rule (@rules) {
+  # Truncate the letters and phones in the rule, while we
+  # can, to get more general rules; if the more general rule
+  # exists, put out the pair.
+  @A = split(",", $rule);
+  @suffixa = split("", $A[0]);
+  @suffixb = split("", $A[1]);
+  @psuffixa = split(" ", $A[2]);
+  @psuffixb = split(" ", $A[3]);
+  for ($common_suffix_len = 0; $common_suffix_len < @suffixa && $common_suffix_len < @suffixb;) {
+    if ($suffixa[$common_suffix_len] eq $suffixb[$common_suffix_len]) {
+      $common_suffix_len++;
+    } else {
+      last;
+    }
+  }
+  for ($common_psuffix_len = 0; $common_psuffix_len < @psuffixa && $common_psuffix_len < @psuffixb;) {
+    if ($psuffixa[$common_psuffix_len] eq $psuffixb[$common_psuffix_len]) {
+      $common_psuffix_len++;
+    } else {
+      last;
+    }
+  }
+  # Get all combinations of pairs of integers <= (common_suffix_len, common_psuffix_len),
+  # except (0,0), and print out this rule together with the corresponding rule (if it exists).
+  for ($m = 0; $m <= $common_suffix_len; $m++) {
+    $sa = join("", @suffixa[$m...$#suffixa]); # @x[a..b] is array slice notation.
+    $sb = join("", @suffixb[$m...$#suffixb]);
+    for ($n = 0; $n <= $common_psuffix_len; $n++) {
+      if (!($m == 0 && $n == 0)) {
+        $psa = join(" ", @psuffixa[$n...$#psuffixa]);
+        $psb = join(" ", @psuffixb[$n...$#psuffixb]);
+        $more_general_rule = join(",", ($sa, $sb, $psa, $psb));
+        if (defined $isrule{$more_general_rule}) {
+          print $rule . ";" . $more_general_rule . "\n";
+        }
+      }
+    }
+  }
+}
+
--- a/egs/aurora4/s5/local/dict/get_rules.pl
+++ b/egs/aurora4/s5/local/dict/get_rules.pl
@ -0,0 +1,204 @@
+#!/usr/bin/perl
+
+# This program creates suggested suffix rules from a dictionary.
+# It outputs quadruples of the form:
+# suffix,base-suffix,psuffix,base-psuffix
+# where "suffix" is the suffix of the letters of a word, "base-suffix" is
+# the suffix of the letters of the base-word, "psuffix" is the suffix of the
+# pronunciation of the word (a space-separated list of phonemes), and
+# "base-psuffix" is the suffix of the pronunciation of the baseword.
+# As far as this program is concerned, there is no distinction between
+# "word" and "base-word".  To simplify things slightly, what it does
+# is return all tuples (a,b,c,d) [with a != b] such that there are
+# at least $min_suffix_count instances in the dictionary of
+# a (word-prefix, pron-prefix) pair where there exists (word,pron)
+# pairs of the form
+# ( word-prefix . a,  pron-prefix . c)
+# and 
+# ( word-prefix . b, pron-prefix . d)
+# For example if (a,b,c,d) equals (USLY,US,S L IY0,S)
+# then this quadruple will be output as long as there at least
+# e.g. 30 instances of prefixes like (FAM, F EY1 M AH0)
+# where there exist (word, pron) pairs like:
+# FAMOUS, F EY1 M AH0 S
+# FAMOUSLY  F EY1 M AH0 S L IY0
+#
+# There are some modifications to the picture above, for efficiency.
+# If $disallow_empty_suffix != 0, this program will not output 4-tuples where
+# the first element (the own-word suffix) is empty, as this would cause
+# efficiency problems in get_candidate_prons.pl.  If 
+# $ignore_prefix_stress != 0, this program will ignore stress markings
+# while evaluating whether prefixes are the same.
+# The minimum count for a quadruple to be output is $min_suffix_count
+# (e.g. 30).
+#
+# The function of this program is not to evaluate the accuracy of these rules;
+# it is mostly a pruning step, where we suggest rules that have large enough
+# counts to be suitable for our later procedure where we evaluate their
+# accuracy in predicting prons.
+
+$disallow_empty_suffix = 1; # Disallow rules where the suffix of the "own-word" is
+   # empty.  This is for efficiency in later stages (e.g. get_candidate_prons.pl).
+$min_prefix_len = 3;  # this must match with get_candidate_prons.pl
+$ignore_prefix_stress = 1; # or 0 to take account of stress in prefix.
+$min_suffix_count = 20;
+
+# Takes in dictionary.
+
+print STDERR "Reading dict\n";
+while(<>) {
+  @A = split(" ", $_);
+  my $word = shift @A;
+  my $pron = join(" ", @A);
+  if (!defined $prons{$word}) {
+    $prons{$word} = $pron;
+    push @words, $word;
+  } else {
+    $prons{$word} = $prons{$word} . ";" . $pron;
+  }
+}
+
+# Get common suffixes (e.g., count >100).  Include empty suffix.
+
+print STDERR "Getting common suffix counts.\n";
+{
+  foreach $word (@words) {
+    $len = length($word);
+    for ($x = $min_prefix_len; $x <= $len; $x++) {
+      $suffix_count{substr($word, $x)}++;
+    }
+  }
+
+  foreach $suffix (keys %suffix_count) {
+    if ($suffix_count{$suffix} >= $min_suffix_count) {
+      $newsuffix_count{$suffix} = $suffix_count{$suffix};
+    }
+  }
+  %suffix_count = %newsuffix_count;
+  undef %newsuffix_count;
+
+  foreach $suffix ( sort { $suffix_count{$b} <=> $suffix_count{$a} } keys %suffix_count ) {
+    print STDERR "$suffix_count{$suffix} $suffix\n";
+  }
+}
+
+print STDERR "Getting common suffix pairs.\n";
+
+{
+  print STDERR " Getting map from prefix -> suffix-set.\n";
+
+  # Create map from prefix -> suffix-set.
+  foreach $word (@words) {
+    $len = length($word);
+    for ($x = $min_prefix_len; $x <= $len; $x++) {
+      $prefix = substr($word, 0, $x);
+      $suffix = substr($word, $x);
+      if (defined $suffix_count{$suffix}) { # Suffix is common...
+        if (!defined $suffixes_of{$prefix}) {
+          $suffixes_of{$prefix} = [ $suffix ]; # Create a reference to a new array with
+          # one element.
+        } else {
+          push @{$suffixes_of{$prefix}}, $suffix; # Push $suffix onto array that the
+          # hash member is a reference .
+        }
+      }
+    }
+  }
+  my %suffix_set_count;
+  print STDERR " Getting map from suffix-set -> count.\n";
+  while ( my ($key, $value) = each(%suffixes_of) ) { 
+    my @suffixes = sort ( @$value );
+    $suffix_set_count{join(";", @suffixes)}++;
+  }
+  print STDERR " Getting counts for suffix pairs.\n";
+  while ( my ($suffix_set, $count) = each (%suffix_set_count) ) {
+    my @suffixes = split(";", $suffix_set);
+    # Consider pairs to be ordered.  This is more convenient
+    # later on.
+    foreach $suffix_a (@suffixes) {
+      foreach $suffix_b (@suffixes) {
+        if ($suffix_a ne $suffix_b) {
+          $suffix_pair = $suffix_a . "," . $suffix_b;
+          $suffix_pair_count{$suffix_pair} += $count;
+        }
+      }
+    }
+  }
+
+  # To save memory, only keep pairs above threshold in the hash.
+  while ( my ($suffix_pair, $count) = each (%suffix_pair_count) ) {
+    if ($count >= $min_suffix_count) {
+      $new_hash{$suffix_pair} = $count;
+    }
+  }
+  %suffix_pair_count = %new_hash;
+  undef %new_hash;
+
+  # Print out the suffix pairs so the user can see.
+  foreach $suffix_pair ( 
+      sort { $suffix_pair_count{$b} <=> $suffix_pair_count{$a} } keys %suffix_pair_count ) {
+    print STDERR "$suffix_pair_count{$suffix_pair} $suffix_pair\n";
+  }
+}
+
+print STDERR "Getting common suffix/suffix/psuffix/psuffix quadruples\n";
+
+{
+  while ( my ($prefix, $suffixes_ref) = each(%suffixes_of) ) {
+    # Note: suffixes_ref is a reference to an array.  We dereference with
+    # @$suffixes_ref.
+    # Consider each pair of suffixes (in each order).
+    foreach my $suffix_a ( @$suffixes_ref ) {
+      foreach my $suffix_b ( @$suffixes_ref ) {
+        # could just used "defined" in next line, but this is for clarity.
+        $suffix_pair = $suffix_a.",".$suffix_b;
+        if ( $suffix_pair_count{$suffix_pair} >= $min_suffix_count ) {
+          foreach $pron_a_str (split(";", $prons{$prefix.$suffix_a})) {
+            @pron_a = split(" ", $pron_a_str);
+            foreach $pron_b_str (split(";", $prons{$prefix.$suffix_b})) {
+              @pron_b = split(" ", $pron_b_str);
+              $len_a = @pron_a; # evaluating array as scalar automatically gives length.
+              $len_b = @pron_b;
+              for (my $pos = 0; $pos <= $len_a && $pos <= $len_b; $pos++) {
+                # $pos is starting-pos of psuffix-pair. 
+                $psuffix_a = join(" ", @pron_a[$pos...$#pron_a]);
+                $psuffix_b = join(" ", @pron_b[$pos...$#pron_b]);
+                $quadruple = $suffix_pair . "," . $psuffix_a . "," . $psuffix_b;
+                $quadruple_count{$quadruple}++;
+                
+                my $pron_a_pos = $pron_a[$pos], $pron_b_pos = $pron_b[$pos];
+                if ($ignore_prefix_stress) {
+                  $pron_a_pos =~ s/\d//; # e.g convert IH0 to IH.  Only affects
+                  $pron_b_pos =~ s/\d//; # whether we exit the loop below.
+                }
+                if ($pron_a_pos ne $pron_b_pos) {
+                  # This is important: we don't consider a pron suffix-pair to be
+                  # valid unless the pron prefix is the same.
+                  last;
+                }
+              }
+            }
+          }
+        }
+      }
+    }
+  }
+  # To save memory, only keep pairs above threshold in the hash.
+  while ( my ($quadruple, $count) = each (%quadruple_count) ) {
+    if ($count >= $min_suffix_count) {
+      $new_hash{$quadruple} = $count;
+    }
+  }
+  %quadruple_count = %new_hash;
+  undef %new_hash;
+  
+  # Print out the quadruples for diagnostics.
+  foreach $quadruple ( 
+    sort { $quadruple_count{$b} <=> $quadruple_count{$a} } keys %quadruple_count ) {
+    print STDERR "$quadruple_count{$quadruple} $quadruple\n";
+  }
+}
+# Now print out the quadruples; these are the output of this program.
+foreach $quadruple (keys %quadruple_count) {
+  print $quadruple."\n";
+}
--- a/egs/aurora4/s5/local/dict/limit_candidate_prons.pl
+++ b/egs/aurora4/s5/local/dict/limit_candidate_prons.pl
@ -0,0 +1,103 @@
+#!/usr/bin/perl
+
+# This program enforces the rule that
+# if a "more specific" rule applies, we cannot use the more general rule.
+# It takes in tuples generated by get_candidate_prons (one per line, separated
+# by ";"), of the form:
+# word;pron;base-word;base-pron;rule-name;de-stress[;rule-score]
+# [note: we mean that the last element, the numeric score of the rule, is optional]
+# and it outputs a (generally shorter) list
+# of the same form.
+
+
+# For each word:
+  # For each (base-word,base-pron):
+  #  Eliminate "more-general" rules as follows:
+  #    For each pair of rules applying to this (base-word, base-pron):
+  #      If pair is in more-general hash, disallow more general one.
+  # Let the output be: for each (base-word, base-pron, rule):
+  # for (destress-prefix) in [yes, no], do:
+  #  print out the word input, the rule-name, [destressed:yes|no], and the new pron.
+
+
+if (@ARGV != 1 && @ARGV != 2) {
+  die "Usage: limit_candidate_prons.pl rule_hierarchy [candidate_prons] > limited_candidate_prons";
+}
+
+$hierarchy = shift @ARGV;
+open(H, "<$hierarchy") || die "Opening rule hierarchy $hierarchy";
+
+while(<H>) {
+  chop;
+  m:.+;.+: || die "Bad rule-hierarchy line $_";
+  $hierarchy{$_} = 1; # Format is: if $rule1 is the string form of the more specific rule
+  # and $rule21 is that string form of the more general rule, then $hierarchy{$rule1.";".$rule2}
+  # is defined, else undefined.
+}
+
+
+sub process_word;
+
+undef $cur_word;
+@cur_lines = ();
+
+while(<>) {
+  # input, output is:
+  # word;pron;base-word;base-pron;rule-name;destress;score
+  chop;
+  m:^([^;]+);: || die "Unexpected input: $_";
+  $word = $1;
+  if (!defined $cur_word || $word eq $cur_word) {
+    if (!defined $cur_word) { $cur_word = $word; }
+    push @cur_lines, $_;
+  } else {
+    process_word(@cur_lines); # Process a series of suggested prons
+    # for a particular word.
+    $cur_word = $word;
+    @cur_lines = ( $_ ); 
+  }
+}
+process_word(@cur_lines);
+  
+sub process_word {
+  my %pair2rule_list; # hash from $baseword.";".$baseword to ref
+  # to array of [ line1, line2, ... ].
+  my @cur_lines = @_;
+  foreach my $line (@cur_lines) {
+    my ($word, $pron, $baseword, $basepron, $rulename, $destress, $rule_score) = split(";", $line);
+    my $key = $baseword.";".$basepron;
+    if (defined $pair2rule_list{$key}) {
+      push @{$pair2rule_list{$key}}, $line; # @{...} derefs the array pointed to 
+      # by the array ref inside {}. 
+    } else {
+      $pair2rule_list{$key} = [ $line ]; # [ $x ] is new anonymous array with 1 elem ($x)
+    }
+  }
+  while ( my ($key, $value) = each(%pair2rule_list) ) {
+    my @lines = @$value; # array of lines that are for this (baseword,basepron).
+    my @stress, @rules; # Arrays of stress markers and rule names, indexed by
+    # same index that indexes @lines.
+    for (my $n = 0; $n < @lines; $n++) {
+      my $line = $lines[$n];
+      my ($word, $pron, $baseword, $basepron, $rulename, $destress, $rule_score) = split(";", $line);
+      $stress[$n] = $destress;
+      $rules[$n] = $rulename;
+    }
+    for (my $m = 0; $m < @lines; $m++) {
+      my $ok = 1; # if stays 1, this line is OK.
+      for (my $n = 0; $n < @lines; $n++) {
+        if ($m != $n && $stress[$m] eq $stress[$n]) {
+          if (defined $hierarchy{$rules[$n].";".$rules[$m]}) {
+            # Note: this "hierarchy" variable is defined if $rules[$n] is a more
+            # specific instances of $rules[$m], thus invalidating $rules[$m].
+            $ok = 0;
+            last; # no point iterating further.
+          }
+        }
+      }
+      if ($ok != 0) {
+        print $lines[$m] . "\n";
+      }
+    }
+  }
+}
--- a/egs/aurora4/s5/local/dict/reverse_candidates.pl
+++ b/egs/aurora4/s5/local/dict/reverse_candidates.pl
@ -0,0 +1,50 @@
+#!/usr/bin/perl
+
+# This takes the output of e.g. get_candidate_prons.pl or limit_candidate_prons.pl,
+# which is 7-tuples, one per line, of the form:
+
+# word;pron;base-word;base-pron;rule-name;de-stress;rule-score
+# (where rule-score is somtimes listed as optional, but this
+# program does expect it, since we don't anticipate it being used
+# without it).
+# This program assumes that all the words and prons and rules have
+# come from a reversed dictionary (reverse_dict.pl) where the order
+# of the characters in the words, and the phones in the prons, have
+# been reversed, and it un-reverses them.  That it, the characters
+# in "word" and "base-word", and the phones in "pron" and "base-pron"
+# are reversed; and the rule ("rule-name") is parsed as a 4-tuple,
+# like:
+# suffix,base-suffix,psuffix,base-psuffix
+# so this program reverses the characters in "suffix" and "base-suffix"
+# and the phones (separated by spaces) in "psuffix" and "base-psuffix".
+
+sub reverse_str {
+  $str = shift;
+  return join("", reverse(split("", $str)));
+}
+sub reverse_pron {
+  $str = shift;
+  return join(" ", reverse(split(" ", $str)));
+}
+
+while(<>){ 
+  chop;
+  @A = split(";", $_);
+  @A == 7 || die "Bad input line $_: found $len fields, expected 7.";
+
+  ($word,$pron,$baseword,$basepron,$rule,$destress,$score) = @A;
+  $word = reverse_str($word);
+  $pron = reverse_pron($pron);
+  $baseword = reverse_str($baseword);
+  $basepron = reverse_pron($basepron);
+  @R = split(",", $rule, 4);
+  @R == 4 || die "Bad rule $rule";
+
+  $R[0] = reverse_str($R[0]); # suffix.
+  $R[1] = reverse_str($R[1]); # base-suffix.
+  $R[2] = reverse_pron($R[2]); # pron.
+  $R[3] = reverse_pron($R[3]); # base-pron.
+  $rule = join(",", @R);
+  @A = ($word,$pron,$baseword,$basepron,$rule,$destress,$score);
+  print join(";", @A) . "\n";
+}
--- a/egs/aurora4/s5/local/dict/reverse_dict.pl
+++ b/egs/aurora4/s5/local/dict/reverse_dict.pl
@ -0,0 +1,14 @@
+#!/usr/bin/perl
+
+# Used in conjunction with get_rules.pl
+# example input line: XANTHE  Z AE1 N DH
+# example output line: EHTNAX DH N AE1 Z
+
+while(<>){ 
+  @A = split(" ", $_);
+  $word = shift @A;
+  $word = join("", reverse(split("", $word))); # Reverse letters of word.
+  @A = reverse(@A); # Reverse phones in pron.
+  unshift @A, $word;
+  print join(" ", @A) . "\n";
+}
--- a/egs/aurora4/s5/local/dict/score_prons.pl
+++ b/egs/aurora4/s5/local/dict/score_prons.pl
@ -0,0 +1,50 @@
+#!/usr/bin/perl
+
+# This program takes candidate prons from "get_candidate_prons.pl" or
+# "limit_candidate_prons.pl", and a reference dictionary covering those words,
+# and outputs the same format but with scoring information added (so we go from
+# 6 to 7 fields).  The scoring information says, for each generated pron,
+# whether we have a match, a partial match, or no match, to some word in the
+# dictionary.  A partial match means it's correct except for stress.
+
+# The input is a 6-tuple on each line, like:
+# word;pron;base-word;base-pron;rule-name;de-stress
+#
+# The output is the same except with one more field, the score,
+# which may be "right", "wrong", "partial".
+
+if (@ARGV != 1 && @ARGV != 2) {
+  die "Usage: score_prons.pl reference_dict [candidate_prons] > scored_candidate_prons";
+}
+
+$dict = shift @ARGV;
+open(D, "<$dict") || die "Opening dictionary $dict";
+
+while(<D>) { # Set up some hashes that tell us when
+  # a (word,pron) pair is correct (and the same for
+  # prons with stress information removed).
+  chop;
+  @A = split(" ", $_);
+  $word = shift @A;
+  $pron = join(" ", @A);
+  $pron_nostress = $pron;
+  $pron_nostress =~ s:\d::g;
+  $word_and_pron{$word.";".$pron} = 1;
+  $word_and_pron_nostress{$word.";".$pron_nostress} = 1;
+}
+
+while(<>) {
+  chop;
+  $line = $_;
+  my ($word, $pron, $baseword, $basepron, $rulename, $destress) = split(";", $line);
+  $pron_nostress = $pron;
+  $pron_nostress =~ s:\d::g;
+  if (defined $word_and_pron{$word.";".$pron}) {
+    $score = "right";
+  } elsif (defined $word_and_pron_nostress{$word.";".$pron_nostress}) {
+    $score = "partial";
+  } else {
+    $score = "wrong";
+  }
+  print $line.";".$score."\n";
+}
--- a/egs/aurora4/s5/local/dict/score_rules.pl
+++ b/egs/aurora4/s5/local/dict/score_rules.pl
@ -0,0 +1,52 @@
+#!/usr/bin/perl
+
+# This program takes the output of count_rules.pl, which is tuples
+# of the form
+#
+# rule;destress;right-count;partial-count;wrong-count
+#
+# and outputs lines of the form
+#
+# rule;de-stress;score
+#
+# where the score, between 0 and 1 (1 better), is 
+# equal to:
+#
+# It forms a score between 0 and 1, of the form:
+# ((#correct) +  $partial_score * (#partial)) / (#correct + #partial + #wrong + $ballast)
+#
+# where $partial_score (e.g. 0.8) is the score we assign to a "partial" match,
+# and $ballast is a small number, e.g. 1, that is treated like "extra" wrong scores, to penalize
+# rules with few observations.
+#
+# It outputs all rules that at are at least the
+
+$ballast = 1;
+$partial_score = 0.8;
+$destress_penalty = 1.0e-05; # Give destressed rules a small
+# penalty vs. their no-destress counterparts, so if we
+# have to choose arbitrarily we won't destress (seems safer)>
+
+for ($n = 1; $n <= 4; $n++) {
+  if ($ARGV[0] eq "--ballast") {
+    shift @ARGV;
+    $ballast = shift @ARGV;
+  }
+  if ($ARGV[0] eq "--partial-score") {
+    shift @ARGV;
+    $partial_score = shift @ARGV;
+    ($partial_score >= 0.0 && $partial_score <= 1.0) || die "Invalid partial_score: $partial_score";
+  }
+}
+
+(@ARGV == 0 || @ARGV == 1) || die "Usage: score_rules.pl [--ballast ballast-count] [--partial-score partial-score] [input from count_rules.pl]";
+
+while(<>) {
+  @A = split(";", $_);
+  @A == 5 || die "Bad input line; $_";
+  ($rule,$destress,$right_count,$partial_count,$wrong_count) = @A;
+  $rule_score = ($right_count + $partial_score*$partial_count) / 
+    ($right_count+$partial_count+$wrong_count+$ballast);
+  if ($destress eq "yes") { $rule_score -= $destress_penalty; }
+  print join(";", $rule, $destress, sprintf("%.5f", $rule_score)) . "\n";
+}
--- a/egs/aurora4/s5/local/dict/select_candidate_prons.pl
+++ b/egs/aurora4/s5/local/dict/select_candidate_prons.pl
@ -0,0 +1,84 @@
+#!/usr/bin/perl
+
+# This takes the output of e.g. get_candidate_prons.pl or limit_candidate_prons.pl
+# or reverse_candidates.pl, which is 7-tuples, one per line, of the form:
+#
+# word;pron;base-word;base-pron;rule-name;de-stress;rule-score
+#
+# and selects the most likely prons for the words based on rule
+# score.  It outputs in the same format as the input (thus, it is
+# similar to limit_candidates.pl in its input and output format,
+# except it has a different way of selecting the prons to put out).
+#
+# This script will select the $max_prons best pronunciations for
+# each candidate word, subject to the constraint that no pron should
+# have a rule score worse than $min_rule_score.
+# It first merges the candidates by, if there are multiple candidates
+# generating the same pron, selecting the candidate that had the
+# best associated score.  It then sorts the prons on score and
+# selects the n best prons (but doesn't print out candidates with
+# score beneath the threshold).
+
+
+$max_prons = 4;
+$min_rule_score = 0.35;
+
+
+for ($n = 1; $n <= 3; $n++) {
+  if ($ARGV[0] eq "--max-prons") {
+    shift @ARGV;
+    $max_prons = shift @ARGV;
+  }
+  if ($ARGV[0] eq "--min-rule-score") {
+    shift @ARGV;
+    $min_rule_score = shift @ARGV;
+  }
+}
+
+if (@ARGV != 0 && @ARGV != 1) {
+  die "Usage: select_candidates_prons.pl [candidate_prons] > selected_candidate_prons";
+}
+
+sub process_word;
+
+undef $cur_word;
+@cur_lines = ();
+
+while(<>) {
+  # input, output is:
+  # word;pron;base-word;base-pron;rule-name;destress;score
+  chop;
+  m:^([^;]+);: || die "Unexpected input: $_";
+  $word = $1;
+  if (!defined $cur_word || $word eq $cur_word) {
+    if (!defined $cur_word) { $cur_word = $word; }
+    push @cur_lines, $_;
+  } else {
+    process_word(@cur_lines); # Process a series of suggested prons
+    # for a particular word.
+    $cur_word = $word;
+    @cur_lines = ( $_ ); 
+  }
+}
+process_word(@cur_lines);
+
+
+sub process_word {
+  my %pron2rule_score; # hash from generated pron to rule score for that pron.
+  my %pron2line; # hash from generated pron to best line for that pron.
+  my @cur_lines = @_;
+  foreach my $line (@cur_lines) {
+    my ($word, $pron, $baseword, $basepron, $rulename, $destress, $rule_score) = split(";", $line);
+    if (!defined $pron2rule_score{$pron} ||
+        $rule_score > $pron2rule_score{$pron}) {
+      $pron2rule_score{$pron} = $rule_score;
+      $pron2line{$pron} = $line;
+    }
+  }
+  my @prons = sort { $pron2rule_score{$b} <=> $pron2rule_score{$a} } keys %pron2rule_score;
+  for (my $n = 0; $n < @prons && $n < $max_prons &&
+       $pron2rule_score{$prons[$n]} >= $min_rule_score; $n++) {
+    print $pron2line{$prons[$n]} . "\n";
+  }
+}
+
--- a/egs/aurora4/s5/local/find_transcripts.pl
+++ b/egs/aurora4/s5/local/find_transcripts.pl
@ -0,0 +1,64 @@
+#!/usr/bin/perl
+# Copyright 2010-2011 Microsoft Corporation
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#  http://www.apache.org/licenses/LICENSE-2.0
+#
+# THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED
+# WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
+# MERCHANTABLITY OR NON-INFRINGEMENT.
+# See the Apache 2 License for the specific language governing permissions and
+# limitations under the License.
+
+
+
+# This program takes on its standard input a list of utterance
+# id's, one for each line. (e.g. 4k0c030a is a an utterance id).
+# It takes as
+# Extracts from the dot files the transcripts for a given
+# dataset (represented by a file list).
+# 
+
+@ARGV == 1 || die "find_transcripts.pl dot_files_flist < utterance_ids > transcripts";
+$dot_flist = shift @ARGV;
+
+open(L, "<$dot_flist") || die "Opening file list of dot files: $dot_flist\n";
+while(<L>){
+    chop;
+    m:\S+/(\w{6})00.dot: || die "Bad line in dot file list: $_";
+    $spk = $1;
+    $spk2dot{$spk} = $_;
+}
+
+
+
+while(<STDIN>){ 
+    chop;
+    $uttid = $_;
+    $uttid =~ m:(\w{6})\w\w: || die "Bad utterance id $_";
+    $spk = $1;
+    if($spk ne $curspk) {
+        %utt2trans = { }; # Don't keep all the transcripts in memory...
+        $curspk = $spk;
+        $dotfile = $spk2dot{$spk};
+        defined $dotfile || die "No dot file for speaker $spk\n";
+        open(F, "<$dotfile") || die "Error opening dot file $dotfile\n";
+        while(<F>) {
+            $_ =~ m:(.+)\((\w{8})\)\s*$: || die "Bad line $_ in dot file $dotfile (line $.)\n";
+            $trans = $1;
+            $utt = $2;
+            $utt2trans{$utt} = $trans;
+        }
+    }
+    if(!defined $utt2trans{$uttid}) {
+        print STDERR "No transcript for utterance $uttid (current dot file is $dotfile)\n";
+    } else {
+        print "$uttid $utt2trans{$uttid}\n";
+    }
+}
+
+
--- a/egs/aurora4/s5/local/flist2scp.pl
+++ b/egs/aurora4/s5/local/flist2scp.pl
@ -0,0 +1,31 @@
+#!/usr/bin/perl
+# Copyright 2010-2011 Microsoft Corporation
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#  http://www.apache.org/licenses/LICENSE-2.0
+#
+# THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED
+# WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
+# MERCHANTABLITY OR NON-INFRINGEMENT.
+# See the Apache 2 License for the specific language governing permissions and
+# limitations under the License.
+
+
+# takes in a file list with lines like
+# /mnt/matylda2/data/WSJ1/13-16.1/wsj1/si_dt_20/4k0/4k0c030a.wv1
+# and outputs an scp in kaldi format with lines like
+# 4k0c030a /mnt/matylda2/data/WSJ1/13-16.1/wsj1/si_dt_20/4k0/4k0c030a.wv1
+# (the first thing is the utterance-id, which is the same as the basename of the file.
+
+
+while(<>){
+    m:^\S+/(\w+)\.[wW][vV]1$: || die "Bad line $_";
+    $id = $1;
+    $id =~ tr/A-Z/a-z/;  # Necessary because of weirdness on disk 13-16.1 (uppercase filenames)
+    print "$id $_";
+}
+
--- a/egs/aurora4/s5/local/flist2scp_12.pl
+++ b/egs/aurora4/s5/local/flist2scp_12.pl
@ -0,0 +1,31 @@
+#!/usr/bin/perl
+# Copyright 2010-2011 Microsoft Corporation
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#  http://www.apache.org/licenses/LICENSE-2.0
+#
+# THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED
+# WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
+# MERCHANTABLITY OR NON-INFRINGEMENT.
+# See the Apache 2 License for the specific language governing permissions and
+# limitations under the License.
+
+
+# takes in a file list with lines like
+# /mnt/matylda2/data/WSJ1/13-16.1/wsj1/si_dt_20/4k0/4k0c030a.wv1
+# and outputs an scp in kaldi format with lines like
+# 4k0c030a /mnt/matylda2/data/WSJ1/13-16.1/wsj1/si_dt_20/4k0/4k0c030a.wv1
+# (the first thing is the utterance-id, which is the same as the basename of the file.
+
+
+while(<>){
+    m:^\S+/(\w+)\.[wW][vV][12]$: || die "Bad line $_";
+    $id = $1;
+    $id =~ tr/A-Z/a-z/;  # Necessary because of weirdness on disk 13-16.1 (uppercase filenames)
+    print "$id $_";
+}
+
--- a/egs/aurora4/s5/local/generate_example_kws.sh
+++ b/egs/aurora4/s5/local/generate_example_kws.sh
@ -0,0 +1,110 @@
+#!/bin/bash
+
+# Copyright 2012  Johns Hopkins University (Author: Guoguo Chen)
+# Apache 2.0.
+
+
+if [ $# -ne 2 ]; then
+   echo "Usage: local/generate_example_kws.sh <data-dir> <kws-data-dir>"
+   echo " e.g.: local/generate_example_kws.sh data/test_eval92/ <data/kws>"
+   exit 1;
+fi
+
+datadir=$1;
+kwsdatadir=$2;
+text=$datadir/text;
+
+mkdir -p $kwsdatadir;
+
+# Generate keywords; we generate 20 unigram keywords with at least 20 counts,
+# 20 bigram keywords with at least 10 counts and 10 trigram keywords with at
+# least 5 counts.
+cat $text | perl -e '
+  %unigram = ();
+  %bigram = ();
+  %trigram = ();
+  while(<>) {
+    chomp;
+    @col=split(" ", $_);
+    shift @col;
+    for($i = 0; $i < @col; $i++) {
+      # unigram case
+      if (!defined($unigram{$col[$i]})) {
+        $unigram{$col[$i]} = 0;
+      }
+      $unigram{$col[$i]}++;
+
+      # bigram case
+      if ($i < @col-1) {
+        $word = $col[$i] . " " . $col[$i+1];
+        if (!defined($bigram{$word})) {
+          $bigram{$word} = 0;
+        }
+        $bigram{$word}++;
+      }
+
+      # trigram case
+      if ($i < @col-2) {
+        $word = $col[$i] . " " . $col[$i+1] . " " . $col[$i+2];
+        if (!defined($trigram{$word})) {
+          $trigram{$word} = 0;
+        }
+        $trigram{$word}++;
+      }
+    }
+  }
+
+  $max_count = 100;
+  $total = 20;
+  $current = 0;
+  $min_count = 20;
+  while ($current < $total && $min_count <= $max_count) {
+    foreach $x (keys %unigram) {
+      if ($unigram{$x} == $min_count) {
+        print "$x\n";
+        $unigram{$x} = 0;
+        $current++;
+      }
+      if ($current == $total) {
+        last;
+      }
+    }
+    $min_count++;
+  }
+  
+  $total = 20;
+  $current = 0;
+  $min_count = 4;
+  while ($current < $total && $min_count <= $max_count) {
+    foreach $x (keys %bigram) {
+      if ($bigram{$x} == $min_count) {
+        print "$x\n";
+        $bigram{$x} = 0;
+        $current++;
+      }
+      if ($current == $total) {
+        last;
+      }
+    }
+    $min_count++;
+  }
+  
+  $total = 10;
+  $current = 0;
+  $min_count = 3;
+  while ($current < $total && $min_count <= $max_count) {
+    foreach $x (keys %trigram) {
+      if ($trigram{$x} == $min_count) {
+        print "$x\n";
+        $trigram{$x} = 0;
+        $current++;
+      }
+      if ($current == $total) {
+        last;
+      }
+    }
+    $min_count++;
+  }
+  ' > $kwsdatadir/raw_keywords.txt
+
+echo "Keywords generation succeeded"
--- a/egs/aurora4/s5/local/kws_data_prep.sh
+++ b/egs/aurora4/s5/local/kws_data_prep.sh
@ -0,0 +1,60 @@
+#!/bin/bash
+
+# Copyright 2012  Johns Hopkins University (Author: Guoguo Chen)
+# Apache 2.0.
+
+
+if [ $# -ne 3 ]; then
+   echo "Usage: local/kws_data_prep.sh <lang-dir> <data-dir> <kws-data-dir>"
+   echo " e.g.: local/kws_data_prep.sh data/lang_test_bd_tgpr/ data/test_eval92/ data/kws/"
+   exit 1;
+fi
+
+langdir=$1;
+datadir=$2;
+kwsdatadir=$3;
+
+mkdir -p $kwsdatadir;
+
+# Create keyword id for each keyword
+cat $kwsdatadir/raw_keywords.txt | perl -e '
+  $idx=1;
+  while(<>) {
+    chomp;
+    printf "WSJ-%04d $_\n", $idx;
+    $idx++;
+  }' > $kwsdatadir/keywords.txt
+
+# Map the keywords to integers; note that we remove the keywords that
+# are not in our $langdir/words.txt, as we won't find them anyway...
+cat $kwsdatadir/keywords.txt | \
+  sym2int.pl --map-oov 0 -f 2- $langdir/words.txt | \
+  grep -v " 0 " | grep -v " 0$" > $kwsdatadir/keywords.int
+
+# Compile keywords into FSTs
+transcripts-to-fsts ark:$kwsdatadir/keywords.int ark:$kwsdatadir/keywords.fsts
+
+# Create utterance id for each utterance; Note that by "utterance" here I mean
+# the keys that will appear in the lattice archive. You may have to modify here
+cat $datadir/wav.scp | \
+  awk '{print $1}' | \
+  sort | uniq | perl -e '
+  $idx=1;
+  while(<>) {
+    chomp;
+    print "$_ $idx\n";
+    $idx++;
+  }' > $kwsdatadir/utter_id
+
+# Map utterance to the names that will appear in the rttm file. You have 
+# to modify the commands below accoring to your rttm file. In the WSJ case
+# since each file is an utterance, we assume that the actual file names will 
+# be the "names" in the rttm, so the utterance names map to themselves.
+cat $datadir/wav.scp | \
+  awk '{print $1}' | \
+  sort | uniq | perl -e '
+  while(<>) {
+    chomp;
+    print "$_ $_\n";
+  }' > $kwsdatadir/utter_map;
+echo "Kws data preparation succeeded"
--- a/egs/aurora4/s5/local/ndx2flist.pl
+++ b/egs/aurora4/s5/local/ndx2flist.pl
@ -0,0 +1,62 @@
+#!/usr/bin/perl
+# Copyright 2010-2011 Microsoft Corporation
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#  http://www.apache.org/licenses/LICENSE-2.0
+#
+# THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED
+# WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
+# MERCHANTABLITY OR NON-INFRINGEMENT.
+# See the Apache 2 License for the specific language governing permissions and
+# limitations under the License.
+
+
+# This program takes as its standard input an .ndx file from the WSJ corpus that looks
+# like this:
+#;; File: tr_s_wv1.ndx, updated 04/26/94
+#;;
+#;; Index for WSJ0 SI-short Sennheiser training data
+#;; Data is read WSJ sentences, Sennheiser mic.
+#;; Contains 84 speakers X (~100 utts per speaker MIT/SRI and ~50 utts 
+#;; per speaker TI) = 7236 utts
+#;;
+#11_1_1:wsj0/si_tr_s/01i/01ic0201.wv1
+#11_1_1:wsj0/si_tr_s/01i/01ic0202.wv1
+#11_1_1:wsj0/si_tr_s/01i/01ic0203.wv1
+
+#and as command-line arguments it takes the names of the WSJ disk locations, e.g.:
+#/mnt/matylda2/data/WSJ0/11-1.1 /mnt/matylda2/data/WSJ0/11-10.1  ... etc.
+# It outputs a list of absolute pathnames (it does this by replacing e.g. 11_1_1 with
+# /mnt/matylda2/data/WSJ0/11-1.1.
+# It also does a slight fix because one of the WSJ disks (WSJ1/13-16.1) was distributed with
+# uppercase rather than lower case filenames.
+
+foreach $fn (@ARGV) {
+    $fn =~ m:.+/([0-9\.\-]+)/?$: || die "Bad command-line argument $fn\n";
+    $disk_id=$1; 
+    $disk_id =~ tr/-\./__/; # replace - and . with - so 11-10.1 becomes 11_10_1
+    $fn =~ s:/$::; # Remove final slash, just in case it is present.
+    $disk2fn{$disk_id} = $fn;
+}
+
+while(<STDIN>){
+    if(m/^;/){ next; } # Comment.  Ignore it.
+    else {
+      m/^([0-9_]+):\s*(\S+)$/  || die "Could not parse line $_";
+      $disk=$1;
+      if(!defined $disk2fn{$disk}) {
+          die "Disk id $disk not found";
+      }
+      $filename = $2; # as a subdirectory of the distributed disk.
+      if($disk eq "13_16_1" && `hostname` =~ m/fit.vutbr.cz/) {
+          # The disk 13-16.1 has been uppercased for some reason, on the
+          # BUT system.  This is a fix specifically for that case.
+          $filename =~ tr/a-z/A-Z/; # This disk contains all uppercase filenames.  Why?
+      }
+      print "$disk2fn{$disk}/$filename\n";
+  }
+}
--- a/egs/aurora4/s5/local/nnet2/run_5b.sh
+++ b/egs/aurora4/s5/local/nnet2/run_5b.sh
@ -0,0 +1,69 @@
+#!/bin/bash
+
+
+stage=0
+train_stage=-100
+# This trains only unadapted (just cepstral mean normalized) features,
+# and uses various combinations of VTLN warping factor and time-warping
+# factor to artificially expand the amount of data.
+
+. cmd.sh
+
+. utils/parse_options.sh  # to parse the --stage option, if given
+
+[ $# != 0 ] && echo "Usage: local/run_4b.sh [--stage <stage> --train-stage <train-stage>]" && exit 1;
+
+set -e
+
+if [ $stage -le 0 ]; then 
+  # Create the training data.
+  featdir=`pwd`/mfcc/nnet5b; mkdir -p $featdir
+  fbank_conf=conf/fbank_40.conf
+  echo "--num-mel-bins=40" > $fbank_conf
+  steps/nnet2/get_perturbed_feats.sh --cmd "$train_cmd" \
+    $fbank_conf $featdir exp/perturbed_fbanks_si284 data/train_si284 data/train_si284_perturbed_fbank &
+  steps/nnet2/get_perturbed_feats.sh --cmd "$train_cmd" --feature-type mfcc \
+    conf/mfcc.conf $featdir exp/perturbed_mfcc_si284 data/train_si284 data/train_si284_perturbed_mfcc &
+  wait
+fi
+
+if [ $stage -le 1 ]; then
+  steps/align_fmllr.sh --nj 30 --cmd "$train_cmd" \
+    data/train_si284_perturbed_mfcc data/lang exp/tri4b exp/tri4b_ali_si284_perturbed_mfcc
+fi 
+
+if [ $stage -le 2 ]; then
+  steps/nnet2/train_block.sh --stage "$train_stage" \
+     --cleanup false \
+     --initial-learning-rate 0.01 --final-learning-rate 0.001 \
+     --num-epochs 10 --num-epochs-extra 5 \
+     --cmd "$decode_cmd" \
+     --hidden-layer-dim 1536 \
+     --num-block-layers 3 --num-normal-layers 3 \
+      data/train_si284_perturbed_fbank data/lang exp/tri4b_ali_si284_perturbed_mfcc exp/nnet5b  || exit 1
+fi
+
+if [ $stage -le 3 ]; then # create testing fbank data.
+  featdir=`pwd`/mfcc
+  fbank_conf=conf/fbank_40.conf
+  for x in test_eval92 test_eval93 test_dev93; do 
+    cp -rT data/$x data/${x}_fbank
+    rm -r ${x}_fbank/split* || true
+    steps/make_fbank.sh --fbank-config "$fbank_conf" --nj 8 \
+      --cmd "$train_cmd" data/${x}_fbank exp/make_fbank/$x $featdir  || exit 1;
+    steps/compute_cmvn_stats.sh data/${x}_fbank exp/make_fbank/$x $featdir  || exit 1;
+  done
+fi
+
+if [ $stage -le 4 ]; then
+  steps/nnet2/decode.sh --cmd "$decode_cmd" --nj 10 \
+     exp/tri4b/graph_bd_tgpr data/test_dev93_fbank exp/nnet5b/decode_bd_tgpr_dev93
+
+  steps/nnet2/decode.sh --cmd "$decode_cmd" --nj 8 \
+     exp/tri4b/graph_bd_tgpr data/test_eval92_fbank exp/nnet5b/decode_bd_tgpr_eval92
+fi
+
+
+
+exit 0;
+
--- a/egs/aurora4/s5/local/nnet2/run_5c.sh
+++ b/egs/aurora4/s5/local/nnet2/run_5c.sh
@ -0,0 +1,24 @@
+#!/bin/bash
+
+# This is neural net training on top of adapted 40-dimensional features.
+# 
+
+. ./cmd.sh
+
+( 
+ steps/nnet2/train_tanh.sh \
+   --mix-up 8000 \
+   --initial-learning-rate 0.01 --final-learning-rate 0.001 \
+   --num-hidden-layers 4 --hidden-layer-dim 1024 \
+   --cmd "$decode_cmd" \
+    data/train_si284 data/lang exp/tri4b_ali_si284 exp/nnet5c || exit 1
+  
+  steps/decode_nnet_cpu.sh --cmd "$decode_cmd" --nj 10 \
+    --transform-dir exp/tri4b/decode_bd_tgpr_dev93 \
+     exp/tri4b/graph_bd_tgpr data/test_dev93 exp/nnet5c/decode_bd_tgpr_dev93
+
+  steps/decode_nnet_cpu.sh --cmd "$decode_cmd" --nj 8 \
+    --transform-dir exp/tri4b/decode_bd_tgpr_eval92 \
+     exp/tri4b/graph_bd_tgpr data/test_eval92 exp/nnet5c/decode_bd_tgpr_eval92
+)
+
--- a/egs/aurora4/s5/local/normalize_transcript.pl
+++ b/egs/aurora4/s5/local/normalize_transcript.pl
@ -0,0 +1,59 @@
+#!/usr/bin/perl
+# Copyright 2010-2011 Microsoft Corporation
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#  http://www.apache.org/licenses/LICENSE-2.0
+#
+# THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED
+# WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
+# MERCHANTABLITY OR NON-INFRINGEMENT.
+# See the Apache 2 License for the specific language governing permissions and
+# limitations under the License.
+
+
+# This takes data from the standard input that's unnormalized transcripts in the format
+# 4k2c0308 Of course there isn\'t any guarantee the company will keep its hot hand [misc_noise] 
+# 4k2c030a [loud_breath] And new hardware such as the set of personal computers I\. B\. M\. introduced last week can lead to unexpected changes in the software business [door_slam] 
+# and outputs normalized transcripts.
+# c.f. /mnt/matylda2/data/WSJ0/11-10.1/wsj0/transcrp/doc/dot_spec.doc
+
+@ARGV == 1 ||  die "usage: normalize_transcript.pl noise_word < transcript > transcript2";
+$noise_word = shift @ARGV;
+
+while(<STDIN>) {
+    $_ =~ m:^(\S+) (.+): || die "bad line $_";
+    $utt = $1;
+    $trans = $2;
+    print "$utt";
+    foreach $w (split (" ",$trans)) {
+        $w =~ tr:a-z:A-Z:; # Upcase everything to match the CMU dictionary. .
+        $w =~ s:\\::g;      # Remove backslashes.  We don't need the quoting.
+        $w =~ s:^\%PERCENT$:PERCENT:; # Normalization for Nov'93 test transcripts.
+        $w =~ s:^\.POINT$:POINT:; # Normalization for Nov'93 test transcripts.
+        if($w =~ m:^\[\<\w+\]$:  || # E.g. [<door_slam], this means a door slammed in the preceding word. Delete.
+           $w =~ m:^\[\w+\>\]$:  ||  # E.g. [door_slam>], this means a door slammed in the next word.  Delete.
+           $w =~ m:\[\w+/\]$: ||  # E.g. [phone_ring/], which indicates the start of this phenomenon.
+           $w =~ m:\[\/\w+]$: ||  # E.g. [/phone_ring], which indicates the end of this phenomenon.
+           $w eq "~" ||        # This is used to indicate truncation of an utterance.  Not a word.
+           $w eq ".") {      # "." is used to indicate a pause.  Silence is optional anyway so not much 
+                             # point including this in the transcript.
+            next; # we won't print this word.
+        } elsif($w =~ m:\[\w+\]:) { # Other noises, e.g. [loud_breath].
+            print " $noise_word";
+        } elsif($w =~ m:^\<([\w\']+)\>$:) {
+            # e.g. replace <and> with and.  (the <> means verbal deletion of a word).. but it's pronounced.
+            print " $1";
+        } elsif($w eq "--DASH") {
+            print " -DASH";  # This is a common issue; the CMU dictionary has it as -DASH.
+#        } elsif($w =~ m:(.+)\-DASH$:) { # E.g. INCORPORATED-DASH... seems the DASH gets combined with previous word
+#            print " $1 -DASH";
+        } else {
+            print " $w";
+        }
+    }
+    print "\n";
+}
--- a/egs/aurora4/s5/local/run_basis_fmllr.sh
+++ b/egs/aurora4/s5/local/run_basis_fmllr.sh
@ -0,0 +1,42 @@
+#!/bin/bash
+
+. cmd.sh
+
+mfccdir=mfcc
+
+# Make "per-utterance" versions of the test sets where the speaker
+# information corresponds to utterances-- to demonstrate adaptation on
+# short utterances, particularly for basis fMLLR
+for x in test_eval92 test_eval93 test_dev93 ; do
+  y=${x}_utt
+  rm -r data/$y
+  cp -r data/$x data/$y
+  cat data/$x/utt2spk | awk '{print $1, $1;}' > data/$y/utt2spk;
+  cp data/$y/utt2spk data/$y/spk2utt;
+  steps/compute_cmvn_stats.sh data/$y exp/make_mfcc/$y $mfccdir || exit 1; 
+done
+
+
+ # basis fMLLR experiments.
+ # First a baseline: decode per-utterance with normal fMLLR.
+steps/decode_fmllr.sh --nj 10 --cmd "$decode_cmd" \
+  exp/tri3b/graph_tgpr data/test_dev93_utt exp/tri3b/decode_tgpr_dev93_utt || exit 1;
+steps/decode_fmllr.sh --nj 8 --cmd "$decode_cmd" \
+  exp/tri3b/graph_tgpr data/test_eval92_utt exp/tri3b/decode_tgpr_eval92_utt || exit 1;
+
+ # get the fMLLR basis.
+steps/get_fmllr_basis.sh --cmd "$train_cmd" data/train_si84 data/lang exp/tri3b
+
+ # decoding tri3b with basis fMLLR
+steps/decode_basis_fmllr.sh --nj 10 --cmd "$decode_cmd" \
+  exp/tri3b/graph_tgpr data/test_dev93 exp/tri3b/decode_tgpr_dev93_basis || exit 1;
+steps/decode_basis_fmllr.sh --nj 8 --cmd "$decode_cmd" \
+  exp/tri3b/graph_tgpr data/test_eval92 exp/tri3b/decode_tgpr_eval92_basis || exit 1;
+
+  # The same, per-utterance.
+steps/decode_basis_fmllr.sh --nj 10 --cmd "$decode_cmd" \
+  exp/tri3b/graph_tgpr data/test_dev93_utt exp/tri3b/decode_tgpr_dev93_basis_utt || exit 1;
+steps/decode_basis_fmllr.sh --nj 8 --cmd "$decode_cmd" \
+  exp/tri3b/graph_tgpr data/test_eval92_utt exp/tri3b/decode_tgpr_eval92_basis_utt || exit 1;
+
+
--- a/egs/aurora4/s5/local/run_dnn.sh
+++ b/egs/aurora4/s5/local/run_dnn.sh
@ -0,0 +1,181 @@
+#!/bin/bash
+
+# Copyright 2012-2013  Brno University of Technology (Author: Karel Vesely)
+# Apache 2.0
+
+# In this recipe we build DNN in four stages:
+# 1) Data preparations : the fMLLR features are stored to disk
+# 2) RBM pre-training : in this unsupervised stage we train stack of RBMs, a good starting point for Cross-entropy trainig
+# 3) Frame-level cross-entropy training : in this stage the objective is to classify frames correctly.
+# 4) Sequence-criterion training : in this stage the objective is to classify the whole sequence correctly,
+#     the idea is similar to the 'Discriminative training' in context of GMM-HMMs.
+
+
+. ./cmd.sh ## You'll want to change cmd.sh to something that will work on your system.
+           ## This relates to the queue.
+
+. ./path.sh ## Source the tools/utils (import the queue.pl)
+
+
+
+#false && \
+{
+gmmdir=exp/tri4b
+
+###
+### Generate the alignments of dev93 
+### (held-out set for Cross-entropy training)
+###
+steps/align_fmllr.sh --nj 10 --cmd "$train_cmd" \
+  data/test_dev93 data/lang $gmmdir exp/tri4b_ali_dev93 || exit 1
+
+###
+### Store the fMLLR features, so we can train on them easily
+###
+
+# train si284
+# generate the features
+dir=data-fmllr-tri4b/train_si284
+steps/make_fmllr_feats.sh --nj 20 --cmd "$train_cmd" \
+   --transform-dir exp/tri4b_ali_si284 \
+   $dir data/train_si284 $gmmdir $dir/_log $dir/_data || exit 1
+
+# eval92
+dir=data-fmllr-tri4b/test_eval92
+steps/make_fmllr_feats.sh --nj 8 --cmd "$train_cmd" \
+   --transform-dir exp/tri4b/decode_tgpr_eval92 \
+   $dir data/test_eval92 $gmmdir $dir/_log $dir/_data || exit 1
+
+# dev93 (unsupervised fMLLR)
+# held-out set of Cross-entropy training
+dir=data-fmllr-tri4b/test_dev93
+steps/make_fmllr_feats.sh --nj 10 --cmd "$train_cmd" \
+   --transform-dir exp/tri4b/decode_tgpr_dev93 \
+   $dir data/test_dev93 $gmmdir $dir/_log $dir/_data || exit 1
+}
+
+
+
+###
+### Now we can pre-train stack of RBMs
+###
+#false && \
+{ # Pre-train the DBN
+dir=exp/tri4b_pretrain-dbn
+(tail --pid=$$ -F $dir/_pretrain_dbn.log 2>/dev/null)&
+$cuda_cmd $dir/_pretrain_dbn.log \
+  steps/pretrain_dbn.sh --rbm-iter 3 data-fmllr-tri4b/train_si284 $dir
+}
+
+
+
+###
+### Now we train the DNN optimizing cross-entropy.
+### This will take quite some time.
+###
+
+#false && \
+{ # Train the MLP
+dir=exp/tri4b_pretrain-dbn_dnn
+ali=exp/tri4b_ali
+feature_transform=exp/tri4b_pretrain-dbn/final.feature_transform
+dbn=exp/tri4b_pretrain-dbn/6.dbn
+(tail --pid=$$ -F $dir/_train_nnet.log 2>/dev/null)& 
+$cuda_cmd $dir/_train_nnet.log \
+  steps/train_nnet.sh --feature-transform $feature_transform --dbn $dbn --hid-layers 0 --learn-rate 0.008 \
+  data-fmllr-tri4b/train_si284 data-fmllr-tri4b/test_dev93 data/lang ${ali}_si284 ${ali}_dev93 $dir || exit 1;
+# decode with 'big-dictionary' (reuse HCLG graph)
+steps/decode_nnet.sh --nj 10 --cmd "$decode_cmd" --acwt 0.10 --config conf/decode_dnn.config \
+  exp/tri4b/graph_bd_tgpr data-fmllr-tri4b/test_dev93 $dir/decode_bd_tgpr_dev93 || exit 1;
+steps/decode_nnet.sh --nj 8 --cmd "$decode_cmd" --acwt 0.10 --config conf/decode_dnn.config \
+  exp/tri4b/graph_bd_tgpr data-fmllr-tri4b/test_eval92 $dir/decode_bd_tgpr_eval92 || exit 1;
+}
+
+
+
+###
+### Finally we train using sMBR criterion.
+### We do Stochastic-GD with per-utterance updates. 
+###
+### To get faster convergence, we will re-generate 
+### the lattices after 1st epoch of sMBR.
+###
+
+dir=exp/tri4b_pretrain-dbn_dnn_smbr
+srcdir=exp/tri4b_pretrain-dbn_dnn
+acwt=0.10
+
+# First we need to generate lattices and alignments:
+#false && \
+{
+steps/align_nnet.sh --nj 100 --cmd "$train_cmd" \
+  data-fmllr-tri4b/train_si284 data/lang $srcdir ${srcdir}_ali_si284 || exit 1;
+steps/make_denlats_nnet.sh --nj 100 --cmd "$decode_cmd" \
+  --config conf/decode_dnn.config --acwt $acwt \
+  data-fmllr-tri4b/train_si284 data/lang $srcdir ${srcdir}_denlats_si284  || exit 1;
+}
+# Now we re-train the hybrid by single iteration of sMBR 
+#false && \
+{
+steps/train_nnet_mpe.sh --cmd "$cuda_cmd" --num-iters 1 --acwt $acwt --do-smbr true \
+  data-fmllr-tri4b/train_si284 data/lang $srcdir \
+  ${srcdir}_ali_si284 ${srcdir}_denlats_si284 $dir || exit 1
+}
+# Decode
+#false && \
+{
+for ITER in 1; do
+  # decode dev93 with big dict graph_bd_tgpr
+  steps/decode_nnet.sh --nj 10 --cmd "$decode_cmd" --config conf/decode_dnn.config \
+    --nnet $dir/${ITER}.nnet --acwt $acwt \
+    exp/tri4b/graph_bd_tgpr data-fmllr-tri4b/test_dev93 $dir/decode_dev93_bd_tgpr_it${ITER} || exit 1
+  # decode eval92 with big dict graph_bd_tgpr
+  steps/decode_nnet.sh --nj 8 --cmd "$decode_cmd" --config conf/decode_dnn.config \
+    --nnet $dir/${ITER}.nnet --acwt $acwt \
+    exp/tri4b/graph_bd_tgpr data-fmllr-tri4b/test_eval92 $dir/decode_eval92_bd_tgpr_it${ITER} || exit 1
+done 
+}
+
+
+###
+### Re-generate lattices and run several more iterations of sMBR
+###
+
+dir=exp/tri4b_pretrain-dbn_dnn_smbr_iter1-lats
+srcdir=exp/tri4b_pretrain-dbn_dnn_smbr
+acwt=0.10
+
+# First we need to generate lattices and alignments:
+#false && \
+{
+steps/align_nnet.sh --nj 100 --cmd "$train_cmd" \
+  data-fmllr-tri4b/train_si284 data/lang $srcdir ${srcdir}_ali_si284 || exit 1;
+steps/make_denlats_nnet.sh --nj 100 --cmd "$decode_cmd" \
+  --config conf/decode_dnn.config --acwt $acwt \
+  data-fmllr-tri4b/train_si284 data/lang $srcdir ${srcdir}_denlats_si284  || exit 1;
+}
+# Now we re-train the hybrid by several iterations of sMBR 
+#false && \
+{
+steps/train_nnet_mpe.sh --cmd "$cuda_cmd" --num-iters 4 --acwt $acwt --do-smbr true \
+  data-fmllr-tri4b/train_si284 data/lang $srcdir \
+  ${srcdir}_ali_si284 ${srcdir}_denlats_si284 $dir 
+}
+# Decode
+#false && \
+{
+for ITER in 1 2 3 4; do
+  # decode dev93 with big dict graph_bd_tgpr
+  steps/decode_nnet.sh --nj 10 --cmd "$decode_cmd" --config conf/decode_dnn.config \
+    --nnet $dir/${ITER}.nnet --acwt $acwt \
+    exp/tri4b/graph_bd_tgpr data-fmllr-tri4b/test_dev93 $dir/decode_dev93_bd_tgpr_it${ITER} || exit 1
+  # decode eval92 with big dict graph_bd_tgpr
+  steps/decode_nnet.sh --nj 8 --cmd "$decode_cmd" --config conf/decode_dnn.config \
+    --nnet $dir/${ITER}.nnet --acwt $acwt \
+    exp/tri4b/graph_bd_tgpr data-fmllr-tri4b/test_eval92 $dir/decode_eval92_bd_tgpr_it${ITER} || exit 1
+done 
+}
+
+
+# Getting results [see RESULTS file]
+# for x in exp/*/decode*; do [ -d $x ] && grep WER $x/wer_* | utils/best_wer.sh; done
--- a/egs/aurora4/s5/local/run_fwdbwd.sh
+++ b/egs/aurora4/s5/local/run_fwdbwd.sh
@ -0,0 +1,41 @@
+#prepare reverse lexicon and language model for backwards decoding
+utils/prepare_lang.sh --reverse true data/local/dict "<SPOKEN_NOISE>" data/local/lang_tmp.reverse data/lang.reverse || exit 1;
+utils/reverse_lm.sh data/local/nist_lm/lm_bg_5k.arpa.gz data/lang.reverse data/lang_test_bg_5k.reverse || exit 1;
+utils/reverse_lm_test.sh data/lang_test_bg_5k data/lang_test_bg_5k.reverse || exit 1;
+
+# normal forward decoding
+utils/mkgraph.sh data/lang_test_bg_5k exp/tri2a exp/tri2a/graph_bg5k
+steps/decode_fwdbwd.sh --beam 10.0 --latbeam 4.0 --nj 8 --cmd "$decode_cmd" \
+  exp/tri2a/graph_bg5k data/test_eval92 exp/tri2a/decode_eval92_bg5k_10 || exit 1;
+
+# backward decoding
+utils/mkgraph.sh --reverse data/lang_test_bg_5k.reverse exp/tri2a exp/tri2a/graph_bg5k_r
+steps/decode_fwdbwd.sh --beam 10.0 --latbeam 4.0 --reverse true --nj 8 --cmd "$decode_cmd" \
+  exp/tri2a/graph_bg5k_r data/test_eval92 exp/tri2a/decode_eval92_bg5k_reverse10 || exit 1;
+
+# pingpong decoding
+steps/decode_fwdbwd.sh --beam 10.0 --max-beam 20.0 --reverse true --nj 8 --cmd "$decode_cmd" \
+  --first_pass exp/tri2a/decode_eval92_bg5k_10 exp/tri2a/graph_bg5k_r data/test_eval92 exp/tri2a/decode_eval92_bg5k_pingpong10 || exit 1;
+steps/decode_fwdbwd.sh --beam 10.0 --max-beam 20.0 --nj 8 --cmd "$decode_cmd" \
+  --first_pass exp/tri2a/decode_eval92_bg5k_reverse10 exp/tri2a/graph_bg5k data/test_eval92 exp/tri2a/decode_eval92_bg5k_pongping10 || exit 1;
+
+# same for bigger language models (on machine with 8GB RAM, you can run the whole decoding in 3-4 min without SGE)
+utils/prepare_lang.sh --reverse true data/local/dict_larger "<SPOKEN_NOISE>" data/local/lang_larger.reverse data/lang_bd.reverse || exit;
+utils/reverse_lm.sh --lexicon data/local/dict_larger/lexicon.txt data/local/local_lm/3gram-mincount/lm_pr6.0.gz data/lang_bd.reverse data/lang_test_bd_tgpr.reverse || exit 1;
+utils/reverse_lm_test.sh data/lang_test_bd_tgpr data/lang_test_bd_tgpr.reverse || exit 1;
+
+utils/mkgraph.sh data/lang_test_bd_tgpr exp/tri2a exp/tri2a/graph_bd_tgpr
+steps/decode_fwdbwd.sh --beam 10.0 --latbeam 4.0 --nj 4 --cmd run.pl \
+  exp/tri2a/graph_bd_tgpr data/test_eval92 exp/tri2a/decode_eval92_bdtgpr4_10 || exit 1;
+
+utils/mkgraph.sh --reverse data/lang_test_bd_tgpr.reverse exp/tri2a exp/tri2a/graph_bd_tgpr_r
+steps/decode_fwdbwd.sh --beam 10.0 --latbeam 4.0 --reverse true --nj 4 --cmd run.pl \
+  exp/tri2a/graph_bd_tgpr_r data/test_eval92 exp/tri2a/decode_eval92_bdtgpr4_reverse10 || exit 1;
+
+steps/decode_fwdbwd.sh --beam 10.0 --max-beam 20.0 --reverse true --nj 4 --cmd run.pl \
+  --first_pass exp/tri2a/decode_eval92_bdtgpr4_10 exp/tri2a/graph_bd_tgpr_r data/test_eval92 \
+  exp/tri2a/decode_eval92_bdtgpr4_pingpong10 || exit 1;
+
+steps/decode_fwdbwd.sh --beam 10.0 --max-beam 20.0 --nj 4 --cmd run.pl \
+  --first_pass exp/tri2a/decode_eval92_bdtgpr4_reverse10 exp/tri2a/graph_bd_tgpr data/test_eval92 \
+  exp/tri2a/decode_eval92_bdtgpr4_pongping10 || exit 1;
--- a/egs/aurora4/s5/local/run_mmi_tri2b.sh
+++ b/egs/aurora4/s5/local/run_mmi_tri2b.sh
@ -0,0 +1,60 @@
+#!/bin/bash
+
+. ./cmd.sh
+
+# Train and test MMI (and boosted MMI) on tri2b system.
+steps/make_denlats.sh --sub-split 20 --nj 10 --cmd "$train_cmd" \
+  data/train_si84 data/lang exp/tri2b exp/tri2b_denlats_si84 || exit 1;
+
+# train the basic MMI system.
+steps/train_mmi.sh --cmd "$train_cmd" \
+  data/train_si84 data/lang exp/tri2b_ali_si84 \
+  exp/tri2b_denlats_si84 exp/tri2b_mmi  || exit 1;
+for iter in 3 4; do
+  steps/decode_si.sh --nj 10 --cmd "$decode_cmd" --iter $iter \
+    exp/tri2b/graph_tgpr data/test_dev93 exp/tri2b_mmi/decode_tgpr_dev93_it$iter &
+  steps/decode_si.sh --nj 8 --cmd "$decode_cmd" --iter $iter \
+     exp/tri2b/graph_tgpr data/test_eval92 exp/tri2b_mmi/decode_tgpr_eval92_it$iter &
+done
+
+# MMI with 0.1 boosting factor.
+steps/train_mmi.sh --cmd "$train_cmd" --boost 0.1 \
+  data/train_si84 data/lang exp/tri2b_ali_si84 exp/tri2b_denlats_si84 \
+  exp/tri2b_mmi_b0.1  || exit 1;
+
+for iter in 3 4; do
+  steps/decode_si.sh --nj 10 --cmd "$decode_cmd" --iter $iter \
+    exp/tri2b/graph_tgpr data/test_dev93 exp/tri2b_mmi_b0.1/decode_tgpr_dev93_it$iter &
+  steps/decode_si.sh --nj 8 --cmd "$decode_cmd" --iter $iter \
+     exp/tri2b/graph_tgpr data/test_eval92 exp/tri2b_mmi_b0.1/decode_tgpr_eval92_it$iter &
+done
+
+
+# Train a UBM with 400 components, for fMMI.
+steps/train_diag_ubm.sh --silence-weight 0.5 --nj 10 --cmd "$train_cmd" \
+  400 data/train_si84 data/lang exp/tri2b_ali_si84 exp/dubm2b
+
+ steps/train_mmi_fmmi.sh --boost 0.1 --cmd "$train_cmd" \
+   data/train_si84 data/lang exp/tri2b_ali_si84 exp/dubm2b exp/tri2b_denlats_si84 \
+   exp/tri2b_fmmi_b0.1
+
+ for iter in `seq 3 8`; do 
+   steps/decode_fmmi.sh --nj 10 --cmd "$decode_cmd" --iter $iter \
+     exp/tri2b/graph_tgpr data/test_dev93 exp/tri2b_fmmi_b0.1/decode_tgpr_dev93_it$iter &
+ done
+
+ steps/train_mmi_fmmi.sh --learning-rate 0.005 --boost 0.1 --cmd "$train_cmd" \
+   data/train_si84 data/lang exp/tri2b_ali_si84 exp/dubm2b exp/tri2b_denlats_si84 \
+   exp/tri2b_fmmi_b0.1_lr0.005 || exit 1;
+ for iter in `seq 3 8`; do 
+   steps/decode_fmmi.sh --nj 10 --cmd "$decode_cmd" --iter $iter \
+     exp/tri2b/graph_tgpr data/test_dev93 exp/tri2b_fmmi_b0.1_lr0.005/decode_tgpr_dev93_it$iter &
+ done
+
+ steps/train_mmi_fmmi_indirect.sh --boost 0.1 --cmd "$train_cmd" \
+   data/train_si84 data/lang exp/tri2b_ali_si84 exp/dubm2b exp/tri2b_denlats_si84 \
+   exp/tri2b_fmmi_indirect_b0.1
+ for iter in `seq 3 8`; do 
+   steps/decode_fmmi.sh --nj 10 --cmd "$decode_cmd" --iter $iter \
+      exp/tri2b/graph_tgpr data/test_dev93 exp/tri2b_fmmi_indirect_b0.1/decode_tgpr_dev93_it$iter &
+ done
--- a/egs/aurora4/s5/local/run_mmi_tri4b.sh
+++ b/egs/aurora4/s5/local/run_mmi_tri4b.sh
@ -0,0 +1,50 @@
+#!/bin/bash
+. ./cmd.sh
+
+steps/make_denlats.sh --nj 30 --sub-split 30 --cmd "$train_cmd" \
+  --transform-dir exp/tri4b_ali_si284 \
+  data/train_si284 data/lang exp/tri4b exp/tri4b_denlats_si284 || exit 1;
+
+steps/train_mmi.sh --cmd "$train_cmd" --boost 0.1 \
+  data/train_si284 data/lang exp/tri4b_ali_si284 exp/tri4b_denlats_si284 \
+  exp/tri4b_mmi_b0.1  || exit 1;
+
+steps/decode.sh --nj 10 --cmd "$decode_cmd" --transform-dir exp/tri3b/decode_tgpr_dev93 \
+  exp/tri4b/graph_tgpr data/test_dev93 exp/tri4b_mmi_b0.1/decode_tgpr_dev93
+
+#first, train UBM for fMMI experiments.
+steps/train_diag_ubm.sh --silence-weight 0.5 --nj 30 --cmd "$train_cmd" \
+  600 data/train_si284 data/lang exp/tri4b_ali_si284 exp/dubm4b
+
+# Next, fMMI+MMI.
+steps/train_mmi_fmmi.sh \
+  --boost 0.1 --cmd "$train_cmd" data/train_si284 data/lang exp/tri4b_ali_si284 exp/dubm4b exp/tri4b_denlats_si284 \
+  exp/tri4b_fmmi_a || exit 1;
+
+for iter in 3 4 5 6 7 8; do
+ steps/decode_fmmi.sh --nj 10  --cmd "$decode_cmd" --iter $iter \
+   --transform-dir exp/tri3b/decode_tgpr_dev93  exp/tri4b/graph_tgpr data/test_dev93 \
+  exp/tri4b_fmmi_a/decode_tgpr_dev93_it$iter &
+done
+# decode the last iter with the bd model.
+for iter in 8; do
+ steps/decode_fmmi.sh --nj 10  --cmd "$decode_cmd" --iter $iter \
+   --transform-dir exp/tri3b/decode_bd_tgpr_dev93  exp/tri4b/graph_bd_tgpr data/test_dev93 \
+  exp/tri4b_fmmi_a/decode_bd_tgpr_dev93_it$iter &
+ steps/decode_fmmi.sh --nj 8  --cmd "$decode_cmd" --iter $iter \
+   --transform-dir exp/tri3b/decode_bd_tgpr_eval92  exp/tri4b/graph_bd_tgpr data/test_eval92 \
+  exp/tri4b_fmmi_a/decode_tgpr_eval92_it$iter &
+done
+
+
+# fMMI + mmi with indirect differential.
+steps/train_mmi_fmmi_indirect.sh \
+  --boost 0.1 --cmd "$train_cmd" data/train_si284 data/lang exp/tri4b_ali_si284 exp/dubm4b exp/tri4b_denlats_si284 \
+  exp/tri4b_fmmi_indirect || exit 1;
+
+for iter in 3 4 5 6 7 8; do
+ steps/decode_fmmi.sh --nj 10  --cmd "$decode_cmd" --iter $iter \
+   --transform-dir exp/tri3b/decode_tgpr_dev93  exp/tri4b/graph_tgpr data/test_dev93 \
+  exp/tri4b_fmmi_indirect/decode_tgpr_dev93_it$iter &
+done
+
--- a/egs/aurora4/s5/local/run_nnet_cpu.sh
+++ b/egs/aurora4/s5/local/run_nnet_cpu.sh
@ -0,0 +1,9 @@
+#!/bin/bash
+
+. ./cmd.sh
+
+
+# ...
+
+local/nnet2/run_5c.sh
+
--- a/egs/aurora4/s5/local/run_raw_fmllr.sh
+++ b/egs/aurora4/s5/local/run_raw_fmllr.sh
@ -0,0 +1,66 @@
+#!/bin/bash
+
+
+steps/align_raw_fmllr.sh --nj 10 --cmd "$train_cmd" --use-graphs true \
+    data/train_si84 data/lang exp/tri2b exp/tri2b_ali_si84_raw
+
+steps/train_raw_sat.sh --cmd "$train_cmd" \
+   2500 15000 data/train_si84 data/lang exp/tri2b_ali_si84_raw exp/tri3c || exit 1;
+
+
+mfccdir=mfcc
+for x in test_eval92 test_eval93 test_dev93 ; do
+  y=${x}_utt
+  cp -rT data/$x data/$y
+  cat data/$x/utt2spk | awk '{print $1, $1;}' > data/$y/utt2spk;
+  cp data/$y/utt2spk data/$y/spk2utt;
+  steps/compute_cmvn_stats.sh data/$y exp/make_mfcc/$y $mfccdir || exit 1; 
+done
+
+(
+utils/mkgraph.sh data/lang_test_tgpr exp/tri3c exp/tri3c/graph_tgpr || exit 1;
+steps/decode_raw_fmllr.sh --nj 10 --cmd "$decode_cmd" \
+  exp/tri3c/graph_tgpr data/test_dev93 exp/tri3c/decode_tgpr_dev93 || exit 1;
+steps/decode_raw_fmllr.sh --nj 8 --cmd "$decode_cmd" \
+  exp/tri3c/graph_tgpr data/test_eval92 exp/tri3c/decode_tgpr_eval92 || exit 1;
+
+steps/decode_raw_fmllr.sh --nj 30 --cmd "$decode_cmd" \
+  exp/tri3c/graph_tgpr data/test_dev93_utt exp/tri3c/decode_tgpr_dev93_utt || exit 1;
+steps/decode_raw_fmllr.sh --nj 30 --cmd "$decode_cmd" \
+  exp/tri3c/graph_tgpr data/test_eval92_utt exp/tri3c/decode_tgpr_eval92_utt || exit 1;
+
+steps/decode_raw_fmllr.sh --use-normal-fmllr true --nj 10 --cmd "$decode_cmd" \
+  exp/tri3c/graph_tgpr data/test_dev93 exp/tri3c/decode_tgpr_dev93_2fmllr || exit 1;
+steps/decode_raw_fmllr.sh --use-normal-fmllr true --nj 8 --cmd "$decode_cmd" \
+  exp/tri3c/graph_tgpr data/test_eval92 exp/tri3c/decode_tgpr_eval92_2fmllr || exit 1;
+)&
+
+(
+utils/mkgraph.sh data/lang_test_bd_tgpr exp/tri3c exp/tri3c/graph_bd_tgpr || exit 1; 
+
+steps/decode_raw_fmllr.sh --cmd "$decode_cmd" --nj 8 exp/tri3c/graph_bd_tgpr \
+    data/test_eval92 exp/tri3c/decode_bd_tgpr_eval92 
+ steps/decode_raw_fmllr.sh --cmd "$decode_cmd" --nj 10 exp/tri3c/graph_bd_tgpr \
+   data/test_dev93 exp/tri3c/decode_bd_tgpr_dev93 
+)&
+
+steps/align_fmllr.sh --nj 20 --cmd "$train_cmd" \
+  data/train_si284 data/lang exp/tri3c exp/tri3c_ali_si284 || exit 1;
+
+
+steps/train_raw_sat.sh  --cmd "$train_cmd" \
+  4200 40000 data/train_si284 data/lang exp/tri3c_ali_si284 exp/tri4d || exit 1;
+(
+ utils/mkgraph.sh data/lang_test_tgpr exp/tri4d exp/tri4d/graph_tgpr || exit 1;
+ steps/decode_raw_fmllr.sh --nj 10 --cmd "$decode_cmd" \
+   exp/tri4d/graph_tgpr data/test_dev93 exp/tri4d/decode_tgpr_dev93 || exit 1;
+ steps/decode_raw_fmllr.sh --nj 8 --cmd "$decode_cmd" \
+   exp/tri4d/graph_tgpr data/test_eval92 exp/tri4d/decode_tgpr_eval92 || exit 1;
+) & 
+
+
+wait
+
+
+#for x in exp/tri3{b,c}/decode_tgpr*; do grep WER $x/wer_* | utils/best_wer.sh ; done
+
--- a/egs/aurora4/s5/local/run_rnnlms_sgmm5b.sh
+++ b/egs/aurora4/s5/local/run_rnnlms_sgmm5b.sh
@ -0,0 +1,42 @@
+#!/bin/bash
+
+for test in dev93 eval92; do
+
+  steps/lmrescore.sh --cmd "$decode_cmd" data/lang_test_bd_tgpr data/lang_test_bd_fg \
+    data/test_${test} exp/sgmm5b_mmi_b0.1/decode_bd_tgpr_${test}_it4 exp/sgmm5b_mmi_b0.1/decode_bd_fg_${test}_it4 || exit 1;
+
+
+# Note: for N-best-list generation, choosing the acoustic scale (12) that gave
+# the best WER on this test set.  Ideally we should do this on a dev set.
+
+ # This step interpolates a small RNNLM (with weight 0.25) with the 4-gram LM.
+  steps/rnnlmrescore.sh \
+    --N 100 --cmd "$decode_cmd" --inv-acwt 12 \
+    0.25 data/lang_test_bd_fg data/local/rnnlm.h30.voc10k data/test_${test} \
+    exp/sgmm5b_mmi_b0.1/decode_bd_fg_${test}_it4 exp/sgmm5b_mmi_b0.1/decode_bd_fg_${test}_it4_rnnlm30_0.25  \
+    || exit 1;
+
+  steps/rnnlmrescore.sh \
+    --N 100 --cmd "$decode_cmd" --inv-acwt 12 \
+    0.5 data/lang_test_bd_fg data/local/rnnlm.h100.voc20k data/test_${test} \
+    exp/sgmm5b_mmi_b0.1/decode_bd_fg_${test}_it4 exp/sgmm5b_mmi_b0.1/decode_bd_fg_${test}_it4_rnnlm100_0.5 \
+    || exit 1;
+
+  steps/rnnlmrescore.sh \
+    --N 100 --cmd "$decode_cmd" --inv-acwt 12 \
+    0.5 data/lang_test_bd_fg data/local/rnnlm.h200.voc30k data/test_${test} \
+    exp/sgmm5b_mmi_b0.1/decode_bd_fg_${test}_it4 exp/sgmm5b_mmi_b0.1/decode_bd_fg_${test}_it4_rnnlm200_0.5 \
+    || exit 1;
+
+  steps/rnnlmrescore.sh \
+    --N 100 --cmd "$decode_cmd" --inv-acwt 12 \
+    0.5 data/lang_test_bd_fg data/local/rnnlm.h300.voc40k data/test_${test} \
+    exp/sgmm5b_mmi_b0.1/decode_bd_fg_${test}_it4 exp/sgmm5b_mmi_b0.1/decode_bd_fg_${test}_it4_rnnlm300_0.5 \
+    || exit 1;
+
+  steps/rnnlmrescore.sh \
+    --N 100 --cmd "$decode_cmd" --inv-acwt 12 \
+    0.75 data/lang_test_bd_fg data/local/rnnlm.h300.voc40k data/test_${test} \
+    exp/sgmm5b_mmi_b0.1/decode_bd_fg_${test}_it4 exp/sgmm5b_mmi_b0.1/decode_bd_fg_${test}_it4_rnnlm300_0.75 \
+    || exit 1;
+done
--- a/egs/aurora4/s5/local/run_rnnlms_tri3b.sh
+++ b/egs/aurora4/s5/local/run_rnnlms_tri3b.sh
@ -0,0 +1,64 @@
+#!/bin/bash
+
+. cmd.sh
+
+ # This step interpolates a small RNNLM (with weight 0.25) with the 4-gram LM.
+steps/rnnlmrescore.sh \
+  --N 100 --cmd "$decode_cmd" --inv-acwt 17 \
+  0.25 data/lang_test_bd_fg data/local/rnnlm.h30.voc10k data/test_eval92 \
+  exp/tri3b/decode_bd_tgpr_eval92_fg exp/tri3b/decode_bd_tgpr_eval92_fg_rnnlm30_0.25  \
+  || exit 1;
+
+steps/rnnlmrescore.sh \
+  --N 100 --cmd "$decode_cmd" --inv-acwt 17 \
+  0.5 data/lang_test_bd_fg data/local/rnnlm.h100.voc20k data/test_eval92 \
+  exp/tri3b/decode_bd_tgpr_eval92_fg exp/tri3b/decode_bd_tgpr_eval92_fg_rnnlm100_0.5 \
+  || exit 1;
+
+steps/rnnlmrescore.sh \
+  --N 100 --cmd "$decode_cmd" --inv-acwt 17 \
+  0.5 data/lang_test_bd_fg data/local/rnnlm.h200.voc30k data/test_eval92 \
+  exp/tri3b/decode_bd_tgpr_eval92_fg exp/tri3b/decode_bd_tgpr_eval92_fg_rnnlm200_0.5 \
+  || exit 1;
+
+steps/rnnlmrescore.sh \
+  --N 100 --cmd "$decode_cmd" --inv-acwt 17 \
+  0.5 data/lang_test_bd_fg data/local/rnnlm.h300.voc40k data/test_eval92 \
+  exp/tri3b/decode_bd_tgpr_eval92_fg exp/tri3b/decode_bd_tgpr_eval92_fg_rnnlm300_0.5 \
+  || exit 1;
+
+steps/rnnlmrescore.sh \
+  --N 1000 --cmd "$decode_cmd" --inv-acwt 17 \
+  0.5 data/lang_test_bd_fg data/local/rnnlm.h300.voc40k data/test_eval92 \
+  exp/tri3b/decode_bd_tgpr_eval92_fg exp/tri3b/decode_bd_tgpr_eval92_fg_rnnlm300_0.5_N1000 
+
+dir=exp/tri3b/decode_bd_tgpr_eval92_fg_rnnlm300_0.75_N1000
+rm -rf $dir
+cp -r exp/tri3b/decode_bd_tgpr_eval92_fg_rnnlm300_0.5_N1000 $dir
+steps/rnnlmrescore.sh \
+  --stage 7 --N 1000 --cmd "$decode_cmd" --inv-acwt 17 \
+  0.75 data/lang_test_bd_fg data/local/rnnlm.h300.voc40k data/test_eval92 \
+  exp/tri3b/decode_bd_tgpr_eval92_fg $dir
+
+dir=exp/tri3b/decode_bd_tgpr_eval92_fg_rnnlm300_0.75
+rm -rf $dir
+cp -r exp/tri3b/decode_bd_tgpr_eval92_fg_rnnlm300_0.5 $dir
+steps/rnnlmrescore.sh \
+  --stage 7 --N 100 --cmd "$decode_cmd" --inv-acwt 17 \
+  0.75 data/lang_test_bd_fg data/local/rnnlm.h300.voc40k data/test_eval92 \
+  exp/tri3b/decode_bd_tgpr_eval92_fg $dir
+
+dir=exp/tri3b/decode_bd_tgpr_eval92_fg_rnnlm300_0.25
+rm -rf $dir
+cp -r exp/tri3b/decode_bd_tgpr_eval92_fg_rnnlm300_0.5 $dir
+steps/rnnlmrescore.sh \
+  --stage 7 --N 100 --cmd "$decode_cmd" --inv-acwt 17 \
+  0.25 data/lang_test_bd_fg data/local/rnnlm.h300.voc40k data/test_eval92 \
+  exp/tri3b/decode_bd_tgpr_eval92_fg $dir
+
+steps/rnnlmrescore.sh \
+  --N 10 --cmd "$decode_cmd" --inv-acwt 17 \
+  0.5 data/lang_test_bd_fg data/local/rnnlm.h300.voc40k data/test_eval92 \
+  exp/tri3b/decode_bd_tgpr_eval92_fg exp/tri3b/decode_bd_tgpr_eval92_fg_rnnlm300_0.5_N10 \
+  || exit 1;
+
--- a/egs/aurora4/s5/local/run_sgmm.sh
+++ b/egs/aurora4/s5/local/run_sgmm.sh
@ -0,0 +1,113 @@
+#!/bin/bash
+
+# This script is invoked from ../run.sh
+# It contains some SGMM-related scripts that I am breaking out of the main run.sh for clarity.
+
+. cmd.sh
+
+# SGMM system on si84 data [sgmm5a].  Note: the system we aligned from used the si284 data for
+# training, but this shouldn't have much effect.
+
+(
+  steps/align_fmllr.sh --nj 30 --cmd "$train_cmd" \
+    data/train_si84 data/lang exp/tri4b exp/tri4b_ali_si84 || exit 1;
+
+  steps/train_ubm.sh --cmd "$train_cmd" \
+    400 data/train_si84 data/lang exp/tri4b_ali_si84 exp/ubm5a || exit 1;
+
+  steps/train_sgmm.sh --cmd "$train_cmd" \
+    3500 10000 data/train_si84 data/lang exp/tri4b_ali_si84 \
+    exp/ubm5b/final.ubm exp/sgmm5a || exit 1;
+
+  (
+    utils/mkgraph.sh data/lang_test_tgpr exp/sgmm5a exp/sgmm5a/graph_tgpr
+    steps/decode_sgmm.sh --nj 10 --cmd "$decode_cmd" --transform-dir exp/tri4b/decode_tgpr_dev93 \
+      exp/sgmm5a/graph_tgpr data/test_dev93 exp/sgmm5a/decode_tgpr_dev93
+  ) &
+
+  steps/align_sgmm.sh --nj 30 --cmd "$train_cmd" --transform-dir exp/tri4b_ali_si84 \
+    --use-graphs true --use-gselect true data/train_si84 data/lang exp/sgmm5a exp/sgmm5a_ali_si84 || exit 1;
+  steps/make_denlats_sgmm.sh --nj 30 --sub-split 30 --cmd "$decode_cmd" --transform-dir exp/tri4b_ali_si84 \
+    data/train_si84 data/lang exp/sgmm5a_ali_si84 exp/sgmm5a_denlats_si84
+
+  steps/train_mmi_sgmm.sh --cmd "$decode_cmd" --transform-dir exp/tri4b_ali_si84 --boost 0.1 \
+    data/train_si84 data/lang exp/sgmm5a_ali_si84 exp/sgmm5a_denlats_si84 exp/sgmm5a_mmi_b0.1
+
+  for iter in 1 2 3 4; do
+    steps/decode_sgmm_rescore.sh --cmd "$decode_cmd" --iter $iter \
+      --transform-dir exp/tri4b/decode_tgpr_dev93 data/lang_test_tgpr data/test_dev93 exp/sgmm5a/decode_tgpr_dev93 \
+      exp/sgmm5a_mmi_b0.1/decode_tgpr_dev93_it$iter &
+  done
+
+  steps/train_mmi_sgmm.sh --cmd "$decode_cmd" --transform-dir exp/tri4b_ali_si84 --boost 0.1 \
+   --update-opts "--cov-min-value=0.9" data/train_si84 data/lang exp/sgmm5a_ali_si84 exp/sgmm5a_denlats_si84 exp/sgmm5a_mmi_b0.1_m0.9
+
+  for iter in 1 2 3 4; do
+    steps/decode_sgmm_rescore.sh --cmd "$decode_cmd" --iter $iter \
+      --transform-dir exp/tri4b/decode_tgpr_dev93 data/lang_test_tgpr data/test_dev93 exp/sgmm5a/decode_tgpr_dev93 \
+      exp/sgmm5a_mmi_b0.1_m0.9/decode_tgpr_dev93_it$iter &
+  done
+
+) &
+
+
+(
+# The next commands are the same thing on all the si284 data.
+
+# SGMM system on the si284 data [sgmm5b]
+  steps/train_ubm.sh --cmd "$train_cmd" \
+    600 data/train_si284 data/lang exp/tri4b_ali_si284 exp/ubm5b || exit 1;
+
+  steps/train_sgmm.sh --cmd "$train_cmd" \
+    5500 25000 data/train_si284 data/lang exp/tri4b_ali_si284 \
+    exp/ubm5b/final.ubm exp/sgmm5b || exit 1;
+
+  (
+    utils/mkgraph.sh data/lang_test_tgpr exp/sgmm5b exp/sgmm5b/graph_tgpr
+    steps/decode_sgmm.sh --nj 10 --cmd "$decode_cmd" --transform-dir exp/tri4b/decode_tgpr_dev93 \
+      exp/sgmm5b/graph_tgpr data/test_dev93 exp/sgmm5b/decode_tgpr_dev93
+    steps/decode_sgmm.sh --nj 8 --cmd "$decode_cmd" --transform-dir exp/tri4b/decode_tgpr_eval92 \
+      exp/sgmm5b/graph_tgpr data/test_eval92 exp/sgmm5b/decode_tgpr_eval92
+
+    utils/mkgraph.sh data/lang_test_bd_tgpr exp/sgmm5b exp/sgmm5b/graph_bd_tgpr || exit 1;
+    steps/decode_sgmm.sh --nj 10 --cmd "$decode_cmd" --transform-dir exp/tri4b/decode_bd_tgpr_dev93 \
+      exp/sgmm5b/graph_bd_tgpr data/test_dev93 exp/sgmm5b/decode_bd_tgpr_dev93
+    steps/decode_sgmm.sh --nj 8 --cmd "$decode_cmd" --transform-dir exp/tri4b/decode_bd_tgpr_eval92 \
+      exp/sgmm5b/graph_bd_tgpr data/test_eval92 exp/sgmm5b/decode_bd_tgpr_eval92
+  ) &
+
+  steps/align_sgmm.sh --nj 30 --cmd "$train_cmd" --transform-dir exp/tri4b_ali_si284 \
+    --use-graphs true --use-gselect true data/train_si284 data/lang exp/sgmm5b exp/sgmm5b_ali_si284 
+
+  steps/make_denlats_sgmm.sh --nj 30 --sub-split 30 --cmd "$decode_cmd" --transform-dir exp/tri4b_ali_si284 \
+    data/train_si284 data/lang exp/sgmm5b_ali_si284 exp/sgmm5b_denlats_si284
+
+  steps/train_mmi_sgmm.sh --cmd "$decode_cmd" --transform-dir exp/tri4b_ali_si284 --boost 0.1 \
+    data/train_si284 data/lang exp/sgmm5b_ali_si284 exp/sgmm5b_denlats_si284 exp/sgmm5b_mmi_b0.1
+
+  for iter in 1 2 3 4; do
+    for test in dev93 eval92; do
+      steps/decode_sgmm_rescore.sh --cmd "$decode_cmd" --iter $iter \
+        --transform-dir exp/tri4b/decode_tgpr_${test} data/lang_test_tgpr data/test_${test} exp/sgmm5b/decode_tgpr_${test} \
+        exp/sgmm5b_mmi_b0.1/decode_tgpr_${test}_it$iter &
+
+      steps/decode_sgmm_rescore.sh --cmd "$decode_cmd" --iter $iter \
+        --transform-dir exp/tri4b/decode_bd_tgpr_${test} data/lang_test_bd_tgpr data/test_${test} exp/sgmm5b/decode_bd_tgpr_${test} \
+        exp/sgmm5b_mmi_b0.1/decode_bd_tgpr_${test}_it$iter &
+     done
+  done
+) &
+
+
+
+# Train quinphone SGMM system. 
+
+steps/train_sgmm.sh  --cmd "$train_cmd" \
+   --context-opts "--context-width=5 --central-position=2" \
+   5500 25000 data/train_si284 data/lang exp/tri4b_ali_si284 \
+   exp/ubm5b/final.ubm exp/sgmm5c || exit 1;
+
+# Decode from lattices in exp/sgmm5a/decode_tgpr_dev93.
+steps/decode_sgmm_fromlats.sh --cmd "$decode_cmd"  --transform-dir exp/tri4b/decode_tgpr_dev93 \
+   data/test_dev93 data/lang_test_tgpr exp/sgmm5a/decode_tgpr_dev93 exp/sgmm5c/decode_tgpr_dev93 
+
--- a/egs/aurora4/s5/local/run_sgmm2.sh
+++ b/egs/aurora4/s5/local/run_sgmm2.sh
@ -0,0 +1,148 @@
+#!/bin/bash
+
+# This script is invoked from ../run.sh
+# It contains some SGMM-related scripts that I am breaking out of the main run.sh for clarity.
+
+. cmd.sh
+
+# Note: you might want to try to give the option --spk-dep-weights=false to train_sgmm2.sh;
+# this takes out the "symmetric SGMM" part which is not always helpful.
+
+# SGMM system on si84 data [sgmm5a].  Note: the system we aligned from used the si284 data for
+# training, but this shouldn't have much effect.
+
+(
+  steps/align_fmllr.sh --nj 30 --cmd "$train_cmd" \
+    data/train_si84 data/lang exp/tri4b exp/tri4b_ali_si84 || exit 1;
+
+  steps/train_ubm.sh --cmd "$train_cmd" \
+    400 data/train_si84 data/lang exp/tri4b_ali_si84 exp/ubm5a || exit 1;
+
+  steps/train_sgmm2.sh --cmd "$train_cmd" \
+    7000 9000 data/train_si84 data/lang exp/tri4b_ali_si84 \
+    exp/ubm5a/final.ubm exp/sgmm2_5a || exit 1;
+
+  (
+    utils/mkgraph.sh data/lang_test_tgpr exp/sgmm2_5a exp/sgmm2_5a/graph_tgpr
+    steps/decode_sgmm2.sh --nj 10 --cmd "$decode_cmd" --transform-dir exp/tri4b/decode_tgpr_dev93 \
+      exp/sgmm2_5a/graph_tgpr data/test_dev93 exp/sgmm2_5a/decode_tgpr_dev93
+  ) &
+
+  steps/align_sgmm2.sh --nj 30 --cmd "$train_cmd" --transform-dir exp/tri4b_ali_si84 \
+    --use-graphs true --use-gselect true data/train_si84 data/lang exp/sgmm2_5a exp/sgmm2_5a_ali_si84 || exit 1;
+  steps/make_denlats_sgmm2.sh --nj 30 --sub-split 30 --cmd "$decode_cmd" --transform-dir exp/tri4b_ali_si84 \
+    data/train_si84 data/lang exp/sgmm2_5a_ali_si84 exp/sgmm2_5a_denlats_si84
+
+  steps/train_mmi_sgmm2.sh --cmd "$decode_cmd" --transform-dir exp/tri4b_ali_si84 --boost 0.1 \
+    data/train_si84 data/lang exp/sgmm2_5a_ali_si84 exp/sgmm2_5a_denlats_si84 exp/sgmm2_5a_mmi_b0.1
+
+  for iter in 1 2 3 4; do
+    steps/decode_sgmm2_rescore.sh --cmd "$decode_cmd" --iter $iter \
+      --transform-dir exp/tri4b/decode_tgpr_dev93 data/lang_test_tgpr data/test_dev93 exp/sgmm2_5a/decode_tgpr_dev93 \
+      exp/sgmm2_5a_mmi_b0.1/decode_tgpr_dev93_it$iter &
+  done
+
+  steps/train_mmi_sgmm2.sh --cmd "$decode_cmd" --transform-dir exp/tri4b_ali_si84 --boost 0.1 \
+   --update-opts "--cov-min-value=0.9" data/train_si84 data/lang exp/sgmm2_5a_ali_si84 exp/sgmm2_5a_denlats_si84 exp/sgmm2_5a_mmi_b0.1_m0.9
+
+  for iter in 1 2 3 4; do
+    steps/decode_sgmm2_rescore.sh --cmd "$decode_cmd" --iter $iter \
+      --transform-dir exp/tri4b/decode_tgpr_dev93 data/lang_test_tgpr data/test_dev93 exp/sgmm2_5a/decode_tgpr_dev93 \
+      exp/sgmm2_5a_mmi_b0.1_m0.9/decode_tgpr_dev93_it$iter &
+  done
+
+) &
+
+
+(
+# The next commands are the same thing on all the si284 data.
+
+# SGMM system on the si284 data [sgmm5b]
+  steps/train_ubm.sh --cmd "$train_cmd" \
+    600 data/train_si284 data/lang exp/tri4b_ali_si284 exp/ubm5b || exit 1;
+
+  steps/train_sgmm2.sh --cmd "$train_cmd" \
+   11000 25000 data/train_si284 data/lang exp/tri4b_ali_si284 \
+    exp/ubm5b/final.ubm exp/sgmm2_5b || exit 1;
+
+  (
+    utils/mkgraph.sh data/lang_test_tgpr exp/sgmm2_5b exp/sgmm2_5b/graph_tgpr
+    steps/decode_sgmm2.sh --nj 10 --cmd "$decode_cmd" --transform-dir exp/tri4b/decode_tgpr_dev93 \
+      exp/sgmm2_5b/graph_tgpr data/test_dev93 exp/sgmm2_5b/decode_tgpr_dev93
+    steps/decode_sgmm2.sh --nj 8 --cmd "$decode_cmd" --transform-dir exp/tri4b/decode_tgpr_eval92 \
+      exp/sgmm2_5b/graph_tgpr data/test_eval92 exp/sgmm2_5b/decode_tgpr_eval92
+
+    utils/mkgraph.sh data/lang_test_bd_tgpr exp/sgmm2_5b exp/sgmm2_5b/graph_bd_tgpr || exit 1;
+    steps/decode_sgmm2.sh --nj 10 --cmd "$decode_cmd" --transform-dir exp/tri4b/decode_bd_tgpr_dev93 \
+      exp/sgmm2_5b/graph_bd_tgpr data/test_dev93 exp/sgmm2_5b/decode_bd_tgpr_dev93
+    steps/decode_sgmm2.sh --nj 8 --cmd "$decode_cmd" --transform-dir exp/tri4b/decode_bd_tgpr_eval92 \
+      exp/sgmm2_5b/graph_bd_tgpr data/test_eval92 exp/sgmm2_5b/decode_bd_tgpr_eval92
+  ) &
+
+
+ # This shows how you would build and test a quinphone SGMM2 system, but
+  (
+   steps/train_sgmm2.sh --cmd "$train_cmd" \
+      --context-opts "--context-width=5 --central-position=2" \
+    11000 25000 data/train_si284 data/lang exp/tri4b_ali_si284 \
+     exp/ubm5b/final.ubm exp/sgmm2_5c || exit 1;
+   # Decode from lattices in exp/sgmm2_5b
+    steps/decode_sgmm2_fromlats.sh --cmd "$decode_cmd"  --transform-dir exp/tri4b/decode_tgpr_dev93 \
+       data/test_dev93 data/lang_test_tgpr exp/sgmm2_5b/decode_tgpr_dev93 exp/sgmm2_5c/decode_tgpr_dev93 
+    steps/decode_sgmm2_fromlats.sh --cmd "$decode_cmd"  --transform-dir exp/tri4b/decode_tgpr_eval92 \
+       data/test_eval92 data/lang_test_tgpr exp/sgmm2_5b/decode_tgpr_eval92 exp/sgmm2_5c/decode_tgpr_eval92 
+  ) &
+
+
+  steps/align_sgmm2.sh --nj 30 --cmd "$train_cmd" --transform-dir exp/tri4b_ali_si284 \
+    --use-graphs true --use-gselect true data/train_si284 data/lang exp/sgmm2_5b exp/sgmm2_5b_ali_si284 
+
+  steps/make_denlats_sgmm2.sh --nj 30 --sub-split 30 --cmd "$decode_cmd" --transform-dir exp/tri4b_ali_si284 \
+    data/train_si284 data/lang exp/sgmm2_5b_ali_si284 exp/sgmm2_5b_denlats_si284
+
+  steps/train_mmi_sgmm2.sh --cmd "$decode_cmd" --transform-dir exp/tri4b_ali_si284 --boost 0.1 \
+    data/train_si284 data/lang exp/sgmm2_5b_ali_si284 exp/sgmm2_5b_denlats_si284 exp/sgmm2_5b_mmi_b0.1
+
+  for iter in 1 2 3 4; do
+    for test in eval92; do # dev93
+      steps/decode_sgmm2_rescore.sh --cmd "$decode_cmd" --iter $iter \
+        --transform-dir exp/tri4b/decode_bd_tgpr_${test} data/lang_test_bd_fg data/test_${test} exp/sgmm2_5b/decode_bd_tgpr_${test} \
+        exp/sgmm2_5b_mmi_b0.1/decode_bd_tgpr_${test}_it$iter &
+     done
+  done
+
+  steps/train_mmi_sgmm2.sh --cmd "$decode_cmd" --transform-dir exp/tri4b_ali_si284 --boost 0.1 \
+    --zero-if-disjoint true data/train_si284 data/lang exp/sgmm2_5b_ali_si284 exp/sgmm2_5b_denlats_si284 exp/sgmm2_5b_mmi_b0.1_z
+
+  for iter in 1 2 3 4; do
+    for test in eval92 dev93; do
+      steps/decode_sgmm2_rescore.sh --cmd "$decode_cmd" --iter $iter \
+        --transform-dir exp/tri4b/decode_bd_tgpr_${test} data/lang_test_bd_fg data/test_${test} exp/sgmm2_5b/decode_bd_tgpr_${test} \
+        exp/sgmm2_5b_mmi_b0.1_z/decode_bd_tgpr_${test}_it$iter &
+     done
+  done
+
+) &
+
+wait
+
+# Examples of combining some of the best decodings: SGMM+MMI with
+# MMI+fMMI on a conventional system.
+ 
+local/score_combine.sh data/test_eval92 \
+   data/lang_test_bd_tgpr \
+   exp/tri4b_fmmi_a/decode_tgpr_eval92_it8 \
+   exp/sgmm2_5b_mmi_b0.1/decode_bd_tgpr_eval92_it3 \
+   exp/combine_tri4b_fmmi_a_sgmm2_5b_mmi_b0.1/decode_bd_tgpr_eval92_it8_3
+
+
+# %WER 4.43 [ 250 / 5643, 41 ins, 12 del, 197 sub ] exp/tri4b_fmmi_a/decode_tgpr_eval92_it8/wer_11
+# %WER 3.85 [ 217 / 5643, 35 ins, 11 del, 171 sub ] exp/sgmm2_5b_mmi_b0.1/decode_bd_tgpr_eval92_it3/wer_10
+# combined to:
+# %WER 3.76 [ 212 / 5643, 32 ins, 12 del, 168 sub ] exp/combine_tri4b_fmmi_a_sgmm2_5b_mmi_b0.1/decode_bd_tgpr_eval92_it8_3/wer_12
+
+# Checking MBR decode of baseline:
+cp -r -T exp/sgmm2_5b_mmi_b0.1/decode_bd_tgpr_eval92_it3{,.mbr}
+local/score_mbr.sh data/test_eval92 data/lang_test_bd_tgpr exp/sgmm2_5b_mmi_b0.1/decode_bd_tgpr_eval92_it3.mbr
+# MBR decoding did not seem to help (baseline was 3.85).  I think this is normal at such low WERs.
+%WER 3.86 [ 218 / 5643, 35 ins, 11 del, 172 sub ] exp/sgmm2_5b_mmi_b0.1/decode_bd_tgpr_eval92_it3.mbr/wer_10
--- a/egs/aurora4/s5/local/score.sh
+++ b/egs/aurora4/s5/local/score.sh
@ -0,0 +1,67 @@
+#!/bin/bash
+# Copyright 2012  Johns Hopkins University (Author: Daniel Povey)
+# Apache 2.0
+
+[ -f ./path.sh ] && . ./path.sh
+
+# begin configuration section.
+cmd=run.pl
+stage=0
+decode_mbr=true
+reverse=false
+word_ins_penalty=0.0
+min_lmwt=9
+max_lmwt=20
+#end configuration section.
+
+[ -f ./path.sh ] && . ./path.sh
+. parse_options.sh || exit 1;
+
+if [ $# -ne 3 ]; then
+  echo "Usage: local/score.sh [--cmd (run.pl|queue.pl...)] <data-dir> <lang-dir|graph-dir> <decode-dir>"
+  echo " Options:"
+  echo "    --cmd (run.pl|queue.pl...)      # specify how to run the sub-processes."
+  echo "    --stage (0|1|2)                 # start scoring script from part-way through."
+  echo "    --decode_mbr (true/false)       # maximum bayes risk decoding (confusion network)."
+  echo "    --min_lmwt <int>                # minumum LM-weight for lattice rescoring "
+  echo "    --max_lmwt <int>                # maximum LM-weight for lattice rescoring "
+  echo "    --reverse (true/false)          # score with time reversed features "
+  exit 1;
+fi
+
+data=$1
+lang_or_graph=$2
+dir=$3
+
+symtab=$lang_or_graph/words.txt
+
+for f in $symtab $dir/lat.1.gz $data/text; do
+  [ ! -f $f ] && echo "score.sh: no such file $f" && exit 1;
+done
+
+mkdir -p $dir/scoring/log
+
+cat $data/text | sed 's:<NOISE>::g' | sed 's:<SPOKEN_NOISE>::g' > $dir/scoring/test_filt.txt
+
+$cmd LMWT=$min_lmwt:$max_lmwt $dir/scoring/log/best_path.LMWT.log \
+  lattice-scale --inv-acoustic-scale=LMWT "ark:gunzip -c $dir/lat.*.gz|" ark:- \| \
+  lattice-add-penalty --word-ins-penalty=$word_ins_penalty ark:- ark:- \| \
+  lattice-best-path --word-symbol-table=$symtab \
+    ark:- ark,t:$dir/scoring/LMWT.tra || exit 1;
+
+if $reverse; then
+  for lmwt in `seq $min_lmwt $max_lmwt`; do
+    mv $dir/scoring/$lmwt.tra $dir/scoring/$lmwt.tra.orig
+    awk '{ printf("%s ",$1); for(i=NF; i>1; i--){ printf("%s ",$i); } printf("\n"); }' \
+       <$dir/scoring/$lmwt.tra.orig >$dir/scoring/$lmwt.tra
+  done
+fi
+
+# Note: the double level of quoting for the sed command
+$cmd LMWT=$min_lmwt:$max_lmwt $dir/scoring/log/score.LMWT.log \
+   cat $dir/scoring/LMWT.tra \| \
+    utils/int2sym.pl -f 2- $symtab \| sed 's:\<UNK\>::g' \| \
+    compute-wer --text --mode=present \
+     ark:$dir/scoring/test_filt.txt  ark,p:- ">&" $dir/wer_LMWT || exit 1;
+
+exit 0;
--- a/egs/aurora4/s5/local/score_combine.sh
+++ b/egs/aurora4/s5/local/score_combine.sh
@ -0,0 +1,95 @@
+#!/bin/bash
+
+# Copyright 2013  Arnab Ghoshal
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#  http://www.apache.org/licenses/LICENSE-2.0
+#
+# THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED
+# WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
+# MERCHANTABLITY OR NON-INFRINGEMENT.
+# See the Apache 2 License for the specific language governing permissions and
+# limitations under the License.
+
+
+# Script for system combination using minimum Bayes risk decoding.
+# This calls lattice-combine to create a union of lattices that have been 
+# normalized by removing the total forward cost from them. The resulting lattice
+# is used as input to lattice-mbr-decode. This should not be put in steps/ or 
+# utils/ since the scores on the combined lattice must not be scaled.
+
+# begin configuration section.
+cmd=run.pl
+min_lmwt=9
+max_lmwt=20
+lat_weights=
+#end configuration section.
+
+help_message="Usage: "$(basename $0)" [options] <data-dir> <graph-dir|lang-dir> <decode-dir1> <decode-dir2> [decode-dir3 ... ] <out-dir>
+Options:
+  --cmd (run.pl|queue.pl...)      # specify how to run the sub-processes.
+  --min-lmwt INT                  # minumum LM-weight for lattice rescoring 
+  --max-lmwt INT                  # maximum LM-weight for lattice rescoring
+  --lat-weights STR               # colon-separated string of lattice weights
+";
+
+[ -f ./path.sh ] && . ./path.sh
+. parse_options.sh || exit 1;
+
+if [ $# -lt 5 ]; then
+  printf "$help_message\n";
+  exit 1;
+fi
+
+data=$1
+graphdir=$2
+odir=${@: -1}  # last argument to the script
+shift 2;
+decode_dirs=( $@ )  # read the remaining arguments into an array
+unset decode_dirs[${#decode_dirs[@]}-1]  # 'pop' the last argument which is odir
+num_sys=${#decode_dirs[@]}  # number of systems to combine
+
+symtab=$graphdir/words.txt
+[ ! -f $symtab ] && echo "$0: missing word symbol table '$symtab'" && exit 1;
+[ ! -f $data/text ] && echo "$0: missing reference '$data/text'" && exit 1;
+
+
+mkdir -p $odir/log
+
+for i in `seq 0 $[num_sys-1]`; do
+  model=${decode_dirs[$i]}/../final.mdl  # model one level up from decode dir
+  for f in $model ${decode_dirs[$i]}/lat.1.gz ; do
+    [ ! -f $f ] && echo "$0: expecting file $f to exist" && exit 1;
+  done
+  lats[$i]="\"ark:gunzip -c ${decode_dirs[$i]}/lat.*.gz |\""
+done
+
+mkdir -p $odir/scoring/log
+
+cat $data/text | sed 's:<NOISE>::g' | sed 's:<SPOKEN_NOISE>::g' \
+  > $odir/scoring/test_filt.txt
+
+if [ -z "$lat_weights" ]; then
+  $cmd LMWT=$min_lmwt:$max_lmwt $odir/log/combine_lats.LMWT.log \
+    lattice-combine --inv-acoustic-scale=LMWT ${lats[@]} ark:- \| \
+    lattice-mbr-decode --word-symbol-table=$symtab ark:- \
+    ark,t:$odir/scoring/LMWT.tra || exit 1;
+else
+  $cmd LMWT=$min_lmwt:$max_lmwt $odir/log/combine_lats.LMWT.log \
+    lattice-combine --inv-acoustic-scale=LMWT --lat-weights=$lat_weights \
+    ${lats[@]} ark:- \| \
+    lattice-mbr-decode --word-symbol-table=$symtab ark:- \
+    ark,t:$odir/scoring/LMWT.tra || exit 1;
+fi
+
+$cmd LMWT=$min_lmwt:$max_lmwt $odir/scoring/log/score.LMWT.log \
+  cat $odir/scoring/LMWT.tra \| \
+  utils/int2sym.pl -f 2- $symtab \| sed 's:\<UNK\>::g' \| \
+  compute-wer --text --mode=present \
+  ark:$odir/scoring/test_filt.txt  ark,p:- ">&" $odir/wer_LMWT || exit 1;
+
+exit 0
--- a/egs/aurora4/s5/local/score_mbr.sh
+++ b/egs/aurora4/s5/local/score_mbr.sh
@ -0,0 +1,58 @@
+#!/bin/bash
+
+# Script for minimum bayes risk decoding.
+
+[ -f ./path.sh ] && . ./path.sh;
+
+# begin configuration section.
+cmd=run.pl
+min_lmwt=9
+max_lmwt=20
+#end configuration section.
+
+[ -f ./path.sh ] && . ./path.sh
+. parse_options.sh || exit 1;
+
+if [ $# -ne 3 ]; then
+  echo "Usage: local/score_mbr.sh [--cmd (run.pl|queue.pl...)] <data-dir> <lang-dir|graph-dir> <decode-dir>"
+  echo " Options:"
+  echo "    --cmd (run.pl|queue.pl...)      # specify how to run the sub-processes."
+  echo "    --min_lmwt <int>                # minumum LM-weight for lattice rescoring "
+  echo "    --max_lmwt <int>                # maximum LM-weight for lattice rescoring "
+  exit 1;
+fi
+
+data=$1
+lang_or_graph=$2
+dir=$3
+
+symtab=$lang_or_graph/words.txt
+
+for f in $symtab $dir/lat.1.gz $data/text; do
+  [ ! -f $f ] && echo "score_mbr.sh: no such file $f" && exit 1;
+done
+
+mkdir -p $dir/scoring/log
+
+cat $data/text | sed 's:<NOISE>::g' | sed 's:<SPOKEN_NOISE>::g' > $dir/scoring/test_filt.txt
+
+# We submit the jobs separately, not as an array, because it's hard
+# to get the inverse of the LM scales.
+rm $dir/.error 2>/dev/null
+for inv_acwt in `seq $min_lmwt $max_lmwt`; do
+  acwt=`perl -e "print (1.0/$inv_acwt);"`
+  $cmd $dir/scoring/rescore_mbr.${inv_acwt}.log \
+    lattice-mbr-decode  --acoustic-scale=$acwt --word-symbol-table=$symtab \
+      "ark:gunzip -c $dir/lat.*.gz|" ark,t:$dir/scoring/${inv_acwt}.tra \
+    || touch $dir/.error &
+done
+wait;
+[ -f $dir/.error ] && echo "score_mbr.sh: errror getting MBR outout.";
+     
+
+$cmd LMWT=$min_lmwt:$max_lmwt $dir/scoring/log/score.LMWT.log \
+   cat $dir/scoring/LMWT.tra \| \
+    utils/int2sym.pl -f 2- $symtab \| sed 's:\<UNK\>::g' \| \
+    compute-wer --text --mode=present \
+     ark:$dir/scoring/test_filt.txt  ark,p:- ">" $dir/wer_LMWT || exit 1;
+
--- a/egs/aurora4/s5/local/wsj_data_prep.sh
+++ b/egs/aurora4/s5/local/wsj_data_prep.sh
@ -0,0 +1,201 @@
+#!/bin/bash
+
+# Copyright 2009-2012  Microsoft Corporation  Johns Hopkins University (Author: Daniel Povey)
+# Apache 2.0.
+
+
+if [ $# -le 3 ]; then
+   echo "Arguments should be a list of WSJ directories, see ../run.sh for example."
+   exit 1;
+fi
+
+
+dir=`pwd`/data/local/data
+lmdir=`pwd`/data/local/nist_lm
+mkdir -p $dir $lmdir
+local=`pwd`/local
+utils=`pwd`/utils
+
+. ./path.sh # Needed for KALDI_ROOT
+export PATH=$PATH:$KALDI_ROOT/tools/irstlm/bin
+sph2pipe=$KALDI_ROOT/tools/sph2pipe_v2.5/sph2pipe
+if [ ! -x $sph2pipe ]; then
+   echo "Could not find (or execute) the sph2pipe program at $sph2pipe";
+   exit 1;
+fi
+
+cd $dir
+
+# Make directory of links to the WSJ disks such as 11-13.1.  This relies on the command
+# line arguments being absolute pathnames.
+rm -r links/ 2>/dev/null
+mkdir links/
+ln -s $* links
+
+# Do some basic checks that we have what we expected.
+if [ ! -d links/11-13.1 -o ! -d links/13-34.1 -o ! -d links/11-2.1 ]; then
+  echo "wsj_data_prep.sh: Spot check of command line arguments failed"
+  echo "Command line arguments must be absolute pathnames to WSJ directories"
+  echo "with names like 11-13.1."
+  exit 1;
+fi
+
+# This version for SI-84
+
+cat links/11-13.1/wsj0/doc/indices/train/tr_s_wv1.ndx | \
+ $local/ndx2flist.pl $* | sort | \
+ grep -v -i 11-2.1/wsj0/si_tr_s/401 > train_si84.flist
+
+nl=`cat train_si84.flist | wc -l`
+[ "$nl" -eq 7138 ] || echo "Warning: expected 7138 lines in train_si84.flist, got $nl"
+
+# This version for SI-284
+cat links/13-34.1/wsj1/doc/indices/si_tr_s.ndx \
+ links/11-13.1/wsj0/doc/indices/train/tr_s_wv1.ndx | \
+ $local/ndx2flist.pl  $* | sort | \
+ grep -v -i 11-2.1/wsj0/si_tr_s/401 > train_si284.flist
+
+nl=`cat train_si284.flist | wc -l`
+[ "$nl" -eq 37416 ] || echo "Warning: expected 37416 lines in train_si284.flist, got $nl"
+
+# Now for the test sets.
+# links/13-34.1/wsj1/doc/indices/readme.doc 
+# describes all the different test sets.
+# Note: each test-set seems to come in multiple versions depending
+# on different vocabulary sizes, verbalized vs. non-verbalized
+# pronunciations, etc.  We use the largest vocab and non-verbalized
+# pronunciations.
+# The most normal one seems to be the "baseline 60k test set", which
+# is h1_p0. 
+
+# Nov'92 (333 utts)
+# These index files have a slightly different format;
+# have to add .wv1
+cat links/11-13.1/wsj0/doc/indices/test/nvp/si_et_20.ndx | \
+  $local/ndx2flist.pl $* |  awk '{printf("%s.wv1\n", $1)}' | \
+  sort > test_eval92.flist
+
+# Nov'92 (330 utts, 5k vocab)
+cat links/11-13.1/wsj0/doc/indices/test/nvp/si_et_05.ndx | \
+  $local/ndx2flist.pl $* |  awk '{printf("%s.wv1\n", $1)}' | \
+  sort > test_eval92_5k.flist
+
+# Nov'93: (213 utts)
+# Have to replace a wrong disk-id.
+cat links/13-32.1/wsj1/doc/indices/wsj1/eval/h1_p0.ndx | \
+  sed s/13_32_1/13_33_1/ | \
+  $local/ndx2flist.pl $* | sort > test_eval93.flist
+
+# Nov'93: (213 utts, 5k)
+cat links/13-32.1/wsj1/doc/indices/wsj1/eval/h2_p0.ndx | \
+  sed s/13_32_1/13_33_1/ | \
+  $local/ndx2flist.pl $* | sort > test_eval93_5k.flist
+
+# Dev-set for Nov'93 (503 utts)
+cat links/13-34.1/wsj1/doc/indices/h1_p0.ndx | \
+  $local/ndx2flist.pl $* | sort > test_dev93.flist
+
+# Dev-set for Nov'93 (513 utts, 5k vocab)
+cat links/13-34.1/wsj1/doc/indices/h2_p0.ndx | \
+  $local/ndx2flist.pl $* | sort > test_dev93_5k.flist
+
+
+# Dev-set Hub 1,2 (503, 913 utterances)
+
+# Note: the ???'s below match WSJ and SI_DT, or wsj and si_dt.  
+# Sometimes this gets copied from the CD's with upcasing, don't know 
+# why (could be older versions of the disks).
+find `readlink links/13-16.1`/???1/??_??_20 -print | grep -i ".wv1" | sort > dev_dt_20.flist
+find `readlink links/13-16.1`/???1/??_??_05 -print | grep -i ".wv1" | sort > dev_dt_05.flist
+
+
+# Finding the transcript files:
+for x in $*; do find -L $x -iname '*.dot'; done > dot_files.flist
+
+# Convert the transcripts into our format (no normalization yet)
+for x in train_si84 train_si284 test_eval92 test_eval93 test_dev93 test_eval92_5k test_eval93_5k test_dev93_5k dev_dt_05 dev_dt_20; do
+   $local/flist2scp.pl $x.flist | sort > ${x}_sph.scp
+   cat ${x}_sph.scp | awk '{print $1}' | $local/find_transcripts.pl  dot_files.flist > $x.trans1
+done
+
+# Do some basic normalization steps.  At this point we don't remove OOVs--
+# that will be done inside the training scripts, as we'd like to make the
+# data-preparation stage independent of the specific lexicon used.
+noiseword="<NOISE>";
+for x in train_si84 train_si284 test_eval92 test_eval93 test_dev93 test_eval92_5k test_eval93_5k test_dev93_5k dev_dt_05 dev_dt_20; do
+   cat $x.trans1 | $local/normalize_transcript.pl $noiseword | sort > $x.txt || exit 1;
+done
+ 
+# Create scp's with wav's. (the wv1 in the distribution is not really wav, it is sph.)
+for x in train_si84 train_si284 test_eval92 test_eval93 test_dev93 test_eval92_5k test_eval93_5k test_dev93_5k dev_dt_05 dev_dt_20; do
+  awk '{printf("%s '$sph2pipe' -f wav %s |\n", $1, $2);}' < ${x}_sph.scp > ${x}_wav.scp
+done
+
+# Make the utt2spk and spk2utt files.
+for x in train_si84 train_si284 test_eval92 test_eval93 test_dev93 test_eval92_5k test_eval93_5k test_dev93_5k dev_dt_05 dev_dt_20; do
+   cat ${x}_sph.scp | awk '{print $1}' | perl -ane 'chop; m:^...:; print "$_ $&\n";' > $x.utt2spk
+   cat $x.utt2spk | $utils/utt2spk_to_spk2utt.pl > $x.spk2utt || exit 1;
+done
+
+
+#in case we want to limit lm's on most frequent words, copy lm training word frequency list
+cp links/13-32.1/wsj1/doc/lng_modl/vocab/wfl_64.lst $lmdir
+chmod u+w $lmdir/*.lst # had weird permissions on source.
+
+# The 20K vocab, open-vocabulary language model (i.e. the one with UNK), without
+# verbalized pronunciations.   This is the most common test setup, I understand.
+
+cp links/13-32.1/wsj1/doc/lng_modl/base_lm/bcb20onp.z $lmdir/lm_bg.arpa.gz || exit 1;
+chmod u+w $lmdir/lm_bg.arpa.gz
+
+# trigram would be:
+cat links/13-32.1/wsj1/doc/lng_modl/base_lm/tcb20onp.z | \
+ perl -e 'while(<>){ if(m/^\\data\\/){ print; last;  } } while(<>){ print; }' | \
+ gzip -c -f > $lmdir/lm_tg.arpa.gz || exit 1;
+
+prune-lm --threshold=1e-7 $lmdir/lm_tg.arpa.gz $lmdir/lm_tgpr.arpa || exit 1;
+gzip -f $lmdir/lm_tgpr.arpa || exit 1;
+
+# repeat for 5k language models
+cp links/13-32.1/wsj1/doc/lng_modl/base_lm/bcb05onp.z  $lmdir/lm_bg_5k.arpa.gz || exit 1;
+chmod u+w $lmdir/lm_bg_5k.arpa.gz
+
+# trigram would be: !only closed vocabulary here!
+cp links/13-32.1/wsj1/doc/lng_modl/base_lm/tcb05cnp.z $lmdir/lm_tg_5k.arpa.gz || exit 1;
+chmod u+w $lmdir/lm_tg_5k.arpa.gz
+gunzip $lmdir/lm_tg_5k.arpa.gz
+tail -n 4328839 $lmdir/lm_tg_5k.arpa | gzip -c -f > $lmdir/lm_tg_5k.arpa.gz
+rm $lmdir/lm_tg_5k.arpa
+
+prune-lm --threshold=1e-7 $lmdir/lm_tg_5k.arpa.gz $lmdir/lm_tgpr_5k.arpa || exit 1;
+gzip -f $lmdir/lm_tgpr_5k.arpa || exit 1;
+
+
+if [ ! -f wsj0-train-spkrinfo.txt ] || [ `cat wsj0-train-spkrinfo.txt | wc -l` -ne 134 ]; then
+  rm wsj0-train-spkrinfo.txt
+  ! wget http://www.ldc.upenn.edu/Catalog/docs/LDC93S6A/wsj0-train-spkrinfo.txt && \
+    echo "Getting wsj0-train-spkrinfo.txt from backup location" && \
+    wget --no-check-certificate https://sourceforge.net/projects/kaldi/files/wsj0-train-spkrinfo.txt 
+fi
+
+if [ ! -f wsj0-train-spkrinfo.txt ]; then
+  echo "Could not get the spkrinfo.txt file from LDC website (moved)?"
+  echo "This is possibly omitted from the training disks; couldn't find it." 
+  echo "Everything else may have worked; we just may be missing gender info"
+  echo "which is only needed for VTLN-related diagnostics anyway."
+  exit 1
+fi
+# Note: wsj0-train-spkrinfo.txt doesn't seem to be on the disks but the
+# LDC put it on the web.  Perhaps it was accidentally omitted from the
+# disks.  
+
+cat links/11-13.1/wsj0/doc/spkrinfo.txt \
+    links/13-32.1/wsj1/doc/evl_spok/spkrinfo.txt \
+    links/13-34.1/wsj1/doc/dev_spok/spkrinfo.txt \
+    links/13-34.1/wsj1/doc/train/spkrinfo.txt \
+   ./wsj0-train-spkrinfo.txt  | \
+    perl -ane 'tr/A-Z/a-z/; m/^;/ || print;' | \
+   awk '{print $1, $2}' | grep -v -- -- | sort | uniq > spk2gender
+
+
+echo "Data preparation succeeded"
--- a/egs/aurora4/s5/local/wsj_extend_dict.sh
+++ b/egs/aurora4/s5/local/wsj_extend_dict.sh
@ -0,0 +1,173 @@
+#!/bin/bash
+
+# This script builds a larger word-list and dictionary 
+# than used for the LMs supplied with the WSJ corpus.
+# It uses a couple of strategies to fill-in words in
+# the LM training data but not in CMUdict.  One is
+# to generate special prons for possible acronyms, that
+# just consist of the constituent letters.  The other
+# is designed to handle derivatives of known words
+# (e.g. deriving the pron of a plural from the pron of
+# the base-word), but in a more general, learned-from-data
+# way.
+# It makes use of scripts in local/dict/
+
+if [ $# -ne 1 ]; then
+  echo "Usage: local/wsj_train_lms.sh /foo/bar/WSJ/13-32.1/"
+  exit 1
+fi
+if [ "`basename $1`" != 13-32.1 ]; then
+  echo "Expecting the argument to this script to end in 13-32.1"
+  exit 1
+fi
+
+# e.g.
+#srcdir=/mnt/matylda2/data/WSJ1/13-32.1
+export PATH=$PATH:`pwd`/local/dict/
+srcdir=$1
+mkdir -p data/local/dict_larger
+dir=data/local/dict_larger
+cp data/local/dict/* data/local/dict_larger # Various files describing phones etc.
+  # are there; we just want to copy them as the phoneset is the same.
+rm data/local/dict_larger/lexicon.txt # we don't want this.
+rm data/local/dict_larger/lexiconp.txt # we don't want this either.
+mincount=2 # Minimum count of an OOV we will try to generate a pron for.
+
+[ ! -f data/local/dict/cmudict/cmudict.0.7a ] && echo "CMU dict not in expected place" && exit 1;
+
+# Remove comments from cmudict; print first field; remove
+# words like FOO(1) which are alternate prons: our dict format won't
+# include these markers.
+grep -v ';;;' data/local/dict/cmudict/cmudict.0.7a | 
+ perl -ane 's/^(\S+)\(\d+\)/$1/; print; ' | sort | uniq > $dir/dict.cmu
+
+cat $dir/dict.cmu | awk '{print $1}' | sort | uniq > $dir/wordlist.cmu
+
+echo "Getting training data [this should take at least a few seconds; if not, there's a problem]"
+
+# Convert to uppercase, remove XML-like markings.
+# For words ending in "." that are not in CMUdict, we assume that these
+# are periods that somehow remained in the data during data preparation,
+# and we we replace the "." with "\n".  Note: we found this by looking at
+# oov.counts below (before adding this rule).
+
+touch $dir/cleaned.gz
+if [ `du -m $dir/cleaned.gz | cut -f 1` -eq 73 ]; then
+  echo "Not getting cleaned data in $dir/cleaned.gz again [already exists]";
+else
+ gunzip -c $srcdir/wsj1/doc/lng_modl/lm_train/np_data/{87,88,89}/*.z \
+  | awk '/^</{next}{print toupper($0)}' | perl -e '
+   open(F, "<$ARGV[0]")||die;
+   while(<F>){ chop; $isword{$_} = 1; }
+   while(<STDIN>) { 
+    @A = split(" ", $_); 
+    for ($n = 0; $n < @A; $n++) {
+      $a = $A[$n];
+      if (! $isword{$a} && $a =~ s/^([^\.]+)\.$/$1/) { # nonwords that end in "."
+         # and have no other "." in them: treat as period.
+         print "$a";
+         if ($n+1 < @A) { print "\n"; }
+      } else { print "$a "; }
+    }
+    print "\n";
+  }
+ ' $dir/wordlist.cmu | gzip -c > $dir/cleaned.gz
+fi
+  
+# get unigram counts
+echo "Getting unigram counts"
+gunzip -c $dir/cleaned.gz | tr -s ' ' '\n' | \
+  awk '{count[$1]++} END{for (w in count) { print count[w], w; }}' | sort -nr > $dir/unigrams
+
+cat $dir/unigrams | awk -v dict=$dir/dict.cmu \
+  'BEGIN{while(getline<dict) seen[$1]=1;} {if(!seen[$2]){print;}}' \
+   > $dir/oov.counts
+
+echo "Most frequent unseen unigrams are: "
+head $dir/oov.counts
+
+# Prune away singleton counts, and remove things with numbers in
+# (which should have been normalized) and with no letters at all.
+
+
+cat $dir/oov.counts | awk -v thresh=$mincount '{if ($1 >= thresh) { print $2; }}' \
+  | awk '/[0-9]/{next;} /[A-Z]/{print;}' > $dir/oovlist
+
+# Automatic rule-finding...
+
+# First make some prons for possible acronyms.
+# Note: we don't do this for things like U.K or U.N,
+# or A.B. (which doesn't exist anyway), 
+# as we consider this normalization/spelling errors.
+
+cat $dir/oovlist | local/dict/get_acronym_prons.pl $dir/dict.cmu  > $dir/dict.acronyms
+
+mkdir $dir/f $dir/b # forward, backward directions of rules...
+  # forward is normal suffix
+  # rules, backward is reversed (prefix rules).  These
+  # dirs contain stuff we create while making the rule-based
+  # extensions to the dictionary.
+
+# Remove ; and , from words, if they are present; these
+# might crash our scripts, as they are used as separators there.
+filter_dict.pl $dir/dict.cmu > $dir/f/dict 
+cat $dir/oovlist | filter_dict.pl > $dir/f/oovs
+reverse_dict.pl $dir/f/dict > $dir/b/dict
+reverse_dict.pl $dir/f/oovs > $dir/b/oovs
+
+# The next stage takes a few minutes.
+# Note: the forward stage takes longer, as English is
+# mostly a suffix-based language, and there are more rules
+# that it finds.
+for d in $dir/f $dir/b; do
+ (
+   cd $d
+   cat dict | get_rules.pl 2>get_rules.log >rules
+   get_rule_hierarchy.pl rules >hierarchy
+   awk '{print $1}' dict | get_candidate_prons.pl rules dict | \
+     limit_candidate_prons.pl hierarchy | \
+     score_prons.pl dict | \
+     count_rules.pl >rule.counts
+   # the sort command below is just for convenience of reading.
+   score_rules.pl <rule.counts | sort -t';' -k3,3 -n -r >rules.with_scores
+   get_candidate_prons.pl rules.with_scores dict oovs | \
+     limit_candidate_prons.pl hierarchy > oovs.candidates
+ )  &   
+done 
+wait
+
+# Merge the candidates.
+reverse_candidates.pl $dir/b/oovs.candidates | cat - $dir/f/oovs.candidates | sort > $dir/oovs.candidates
+select_candidate_prons.pl <$dir/oovs.candidates | awk -F';' '{printf("%s  %s\n", $1, $2);}' \
+  > $dir/dict.oovs
+
+cat $dir/dict.acronyms $dir/dict.oovs | sort | uniq > $dir/dict.oovs_merged
+
+awk '{print $1}' $dir/dict.oovs_merged | uniq > $dir/oovlist.handled
+sort $dir/oovlist | diff - $dir/oovlist.handled  | grep -v 'd' | sed 's:< ::' > $dir/oovlist.not_handled
+
+
+# add_counts.pl attaches to original counts to the list of handled/not-handled OOVs
+add_counts.pl $dir/oov.counts $dir/oovlist.handled | sort -nr > $dir/oovlist.handled.counts
+add_counts.pl $dir/oov.counts $dir/oovlist.not_handled | sort -nr > $dir/oovlist.not_handled.counts
+
+echo "**Top OOVs we handled are:**"; 
+head $dir/oovlist.handled.counts
+echo "**Top OOVs we didn't handle are as follows (note: they are mostly misspellings):**"; 
+head $dir/oovlist.not_handled.counts
+
+
+echo "Count of OOVs we handled is `awk '{x+=$1} END{print x}' $dir/oovlist.handled.counts`"
+echo "Count of OOVs we couldn't handle is `awk '{x+=$1} END{print x}' $dir/oovlist.not_handled.counts`"
+echo "Count of OOVs we didn't handle due to low count is" \
+    `awk -v thresh=$mincount '{if ($1 < thresh) x+=$1; } END{print x;}' $dir/oov.counts`
+# The two files created above are for humans to look at, as diagnostics.
+
+cat <<EOF | cat - $dir/dict.cmu $dir/dict.oovs_merged | sort | uniq > $dir/lexicon.txt
+!SIL SIL
+<SPOKEN_NOISE> SPN
+<UNK> SPN
+<NOISE> NSN
+EOF
+
+echo "Created $dir/lexicon.txt"
--- a/egs/aurora4/s5/local/wsj_format_data.sh
+++ b/egs/aurora4/s5/local/wsj_format_data.sh
@ -0,0 +1,86 @@
+#!/bin/bash
+
+# Copyright 2012  Microsoft Corporation  Johns Hopkins University (Author: Daniel Povey)
+# Apache 2.0
+
+# This script takes data prepared in a corpus-dependent way
+# in data/local/, and converts it into the "canonical" form,
+# in various subdirectories of data/, e.g. data/lang, data/lang_test_ug,
+# data/train_si284, data/train_si84, etc.
+
+# Don't bother doing train_si84 separately (although we have the file lists
+# in data/local/) because it's just the first 7138 utterances in train_si284.
+# We'll create train_si84 after doing the feature extraction.
+
+. ./path.sh || exit 1;
+
+echo "Preparing train and test data"
+srcdir=data/local/data
+lmdir=data/local/nist_lm
+tmpdir=data/local/lm_tmp
+lexicon=data/local/lang_tmp/lexiconp.txt
+mkdir -p $tmpdir
+
+for x in train_si284 test_eval92 test_eval93 test_dev93 test_eval92_5k test_eval93_5k test_dev93_5k dev_dt_05 dev_dt_20; do 
+  mkdir -p data/$x
+  cp $srcdir/${x}_wav.scp data/$x/wav.scp || exit 1;
+  cp $srcdir/$x.txt data/$x/text || exit 1;
+  cp $srcdir/$x.spk2utt data/$x/spk2utt || exit 1;
+  cp $srcdir/$x.utt2spk data/$x/utt2spk || exit 1;
+  utils/filter_scp.pl data/$x/spk2utt $srcdir/spk2gender > data/$x/spk2gender || exit 1;
+done
+
+
+# Next, for each type of language model, create the corresponding FST
+# and the corresponding lang_test_* directory.
+
+echo Preparing language models for test
+
+for lm_suffix in bg tgpr tg bg_5k tgpr_5k tg_5k; do
+  test=data/lang_test_${lm_suffix}
+  mkdir -p $test
+  for f in phones.txt words.txt phones.txt L.fst L_disambig.fst \
+     phones/; do
+    cp -r data/lang/$f $test
+  done
+  gunzip -c $lmdir/lm_${lm_suffix}.arpa.gz | \
+   utils/find_arpa_oovs.pl $test/words.txt  > $tmpdir/oovs_${lm_suffix}.txt
+
+  # grep -v '<s> <s>' because the LM seems to have some strange and useless
+  # stuff in it with multiple <s>'s in the history.  Encountered some other similar
+  # things in a LM from Geoff.  Removing all "illegal" combinations of <s> and </s>,
+  # which are supposed to occur only at being/end of utt.  These can cause 
+  # determinization failures of CLG [ends up being epsilon cycles].
+  gunzip -c $lmdir/lm_${lm_suffix}.arpa.gz | \
+    grep -v '<s> <s>' | \
+    grep -v '</s> <s>' | \
+    grep -v '</s> </s>' | \
+    arpa2fst - | fstprint | \
+    utils/remove_oovs.pl $tmpdir/oovs_${lm_suffix}.txt | \
+    utils/eps2disambig.pl | utils/s2eps.pl | fstcompile --isymbols=$test/words.txt \
+      --osymbols=$test/words.txt  --keep_isymbols=false --keep_osymbols=false | \
+     fstrmepsilon > $test/G.fst
+  fstisstochastic $test/G.fst
+ # The output is like:
+ # 9.14233e-05 -0.259833
+ # we do expect the first of these 2 numbers to be close to zero (the second is
+ # nonzero because the backoff weights make the states sum to >1).
+ # Because of the <s> fiasco for these particular LMs, the first number is not
+ # as close to zero as it could be.
+
+  # Everything below is only for diagnostic.
+  # Checking that G has no cycles with empty words on them (e.g. <s>, </s>);
+  # this might cause determinization failure of CLG.
+  # #0 is treated as an empty word.
+  mkdir -p $tmpdir/g
+  awk '{if(NF==1){ printf("0 0 %s %s\n", $1,$1); }} END{print "0 0 #0 #0"; print "0";}' \
+    < "$lexicon"  >$tmpdir/g/select_empty.fst.txt
+  fstcompile --isymbols=$test/words.txt --osymbols=$test/words.txt $tmpdir/g/select_empty.fst.txt | \
+   fstarcsort --sort_type=olabel | fstcompose - $test/G.fst > $tmpdir/g/empty_words.fst
+  fstinfo $tmpdir/g/empty_words.fst | grep cyclic | grep -w 'y' && 
+    echo "Language model has cycles with empty words" && exit 1
+  rm -r $tmpdir/g
+done
+
+echo "Succeeded in formatting data."
+rm -r $tmpdir
--- a/egs/aurora4/s5/local/wsj_format_local_lms.sh
+++ b/egs/aurora4/s5/local/wsj_format_local_lms.sh
@ -0,0 +1,52 @@
+#!/bin/bash
+
+# Copyright Johns Hopkins University (Author: Daniel Povey) 2012
+
+. ./path.sh
+
+[ ! -d data/lang_bd ] && echo "Expect data/local/lang_bd to exist" && exit 1;
+
+lm_srcdir_3g=data/local/local_lm/3gram-mincount
+lm_srcdir_4g=data/local/local_lm/4gram-mincount
+
+[ ! -d "$lm_srcdir_3g" ] && echo "No such dir $lm_srcdir_3g" && exit 1;
+[ ! -d "$lm_srcdir_4g" ] && echo "No such dir $lm_srcdir_4g" && exit 1;
+
+for d in data/lang_test_bd_{tg,tgpr,fg,fgpr}; do
+  rm -r $d 2>/dev/null
+  cp -r data/lang_bd $d
+done
+
+lang=data/lang_bd
+
+# Be careful: this time we dispense with the grep -v '<s> <s>' so this might
+# not work for LMs generated from all toolkits.
+gunzip -c $lm_srcdir_3g/lm_pr6.0.gz | \
+  arpa2fst - | fstprint | \
+    utils/eps2disambig.pl | utils/s2eps.pl | fstcompile --isymbols=$lang/words.txt \
+      --osymbols=$lang/words.txt  --keep_isymbols=false --keep_osymbols=false | \
+     fstrmepsilon > data/lang_test_bd_tgpr/G.fst || exit 1;
+  fstisstochastic data/lang_test_bd_tgpr/G.fst
+
+gunzip -c $lm_srcdir_3g/lm_unpruned.gz | \
+  arpa2fst - | fstprint | \
+    utils/eps2disambig.pl | utils/s2eps.pl | fstcompile --isymbols=$lang/words.txt \
+      --osymbols=$lang/words.txt  --keep_isymbols=false --keep_osymbols=false | \
+     fstrmepsilon > data/lang_test_bd_tg/G.fst || exit 1;
+  fstisstochastic data/lang_test_bd_tg/G.fst
+
+gunzip -c $lm_srcdir_4g/lm_unpruned.gz | \
+  arpa2fst - | fstprint | \
+    utils/eps2disambig.pl | utils/s2eps.pl | fstcompile --isymbols=$lang/words.txt \
+      --osymbols=$lang/words.txt  --keep_isymbols=false --keep_osymbols=false | \
+     fstrmepsilon > data/lang_test_bd_fg/G.fst || exit 1;
+  fstisstochastic data/lang_test_bd_fg/G.fst
+
+gunzip -c $lm_srcdir_4g/lm_pr7.0.gz | \
+  arpa2fst - | fstprint | \
+    utils/eps2disambig.pl | utils/s2eps.pl | fstcompile --isymbols=$lang/words.txt \
+      --osymbols=$lang/words.txt  --keep_isymbols=false --keep_osymbols=false | \
+     fstrmepsilon > data/lang_test_bd_fgpr/G.fst || exit 1;
+  fstisstochastic data/lang_test_bd_fgpr/G.fst
+
+exit 0;
--- a/egs/aurora4/s5/local/wsj_prepare_dict.sh
+++ b/egs/aurora4/s5/local/wsj_prepare_dict.sh
@ -0,0 +1,83 @@
+#!/bin/bash
+
+# Copyright 2010-2012 Microsoft Corporation  Johns Hopkins University (Author: Daniel Povey)
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#  http://www.apache.org/licenses/LICENSE-2.0
+#
+# THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED
+# WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
+# MERCHANTABLITY OR NON-INFRINGEMENT.
+# See the Apache 2 License for the specific language governing permissions and
+# limitations under the License.
+
+# Call this script from one level above, e.g. from the s3/ directory.  It puts
+# its output in data/local/.
+
+# The parts of the output of this that will be needed are
+# [in data/local/dict/ ]
+# lexicon.txt
+# extra_questions.txt
+# nonsilence_phones.txt
+# optional_silence.txt
+# silence_phones.txt
+
+# run this from ../
+dir=data/local/dict
+mkdir -p $dir
+
+
+# (1) Get the CMU dictionary
+svn co  https://svn.code.sf.net/p/cmusphinx/code/trunk/cmudict \
+  $dir/cmudict || exit 1;
+
+# can add -r 10966 for strict compatibility.
+
+
+#(2) Dictionary preparation:
+
+
+# Make phones symbol-table (adding in silence and verbal and non-verbal noises at this point).
+# We are adding suffixes _B, _E, _S for beginning, ending, and singleton phones.
+
+# silence phones, one per line.
+(echo SIL; echo SPN; echo NSN) > $dir/silence_phones.txt
+echo SIL > $dir/optional_silence.txt
+
+# nonsilence phones; on each line is a list of phones that correspond
+# really to the same base phone.
+cat $dir/cmudict/cmudict.0.7a.symbols | perl -ane 's:\r::; print;' | \
+ perl -e 'while(<>){
+  chop; m:^([^\d]+)(\d*)$: || die "Bad phone $_"; 
+  $phones_of{$1} .= "$_ "; }
+  foreach $list (values %phones_of) {print $list . "\n"; } ' \
+  > $dir/nonsilence_phones.txt || exit 1;
+
+# A few extra questions that will be added to those obtained by automatically clustering
+# the "real" phones.  These ask about stress; there's also one for silence.
+cat $dir/silence_phones.txt| awk '{printf("%s ", $1);} END{printf "\n";}' > $dir/extra_questions.txt || exit 1;
+cat $dir/nonsilence_phones.txt | perl -e 'while(<>){ foreach $p (split(" ", $_)) {
+  $p =~ m:^([^\d]+)(\d*)$: || die "Bad phone $_"; $q{$2} .= "$p "; } } foreach $l (values %q) {print "$l\n";}' \
+ >> $dir/extra_questions.txt || exit 1;
+
+grep -v ';;;' $dir/cmudict/cmudict.0.7a | \
+ perl -ane 'if(!m:^;;;:){ s:(\S+)\(\d+\) :$1 :; print; }' \
+  > $dir/lexicon1_raw_nosil.txt || exit 1;
+
+# Add to cmudict the silences, noises etc.
+
+(echo '!SIL SIL'; echo '<SPOKEN_NOISE> SPN'; echo '<UNK> SPN'; echo '<NOISE> NSN'; ) | \
+ cat - $dir/lexicon1_raw_nosil.txt  > $dir/lexicon2_raw.txt || exit 1;
+
+
+# lexicon.txt is without the _B, _E, _S, _I markers.
+# This is the input to wsj_format_data.sh
+cp $dir/lexicon2_raw.txt $dir/lexicon.txt
+
+
+echo "Dictionary preparation succeeded"
+
--- a/egs/aurora4/s5/local/wsj_train_lms.sh
+++ b/egs/aurora4/s5/local/wsj_train_lms.sh
@ -0,0 +1,202 @@
+#!/bin/bash
+
+# This script trains LMs on the WSJ LM-training data.
+# It requires that you have already run wsj_extend_dict.sh,
+# to get the larger-size dictionary including all of CMUdict
+# plus any OOVs and possible acronyms that we could easily 
+# derive pronunciations for.
+
+# This script takes no command-line arguments
+
+dir=data/local/local_lm
+srcdir=data/local/dict_larger
+mkdir -p $dir
+. ./path.sh || exit 1; # for KALDI_ROOT
+export PATH=$KALDI_ROOT/tools/kaldi_lm:$PATH
+( # First make sure the kaldi_lm toolkit is installed.
+ cd $KALDI_ROOT/tools || exit 1;
+ if [ -d kaldi_lm ]; then
+   echo Not installing the kaldi_lm toolkit since it is already there.
+ else
+   echo Downloading and installing the kaldi_lm tools
+   if [ ! -f kaldi_lm.tar.gz ]; then
+     wget http://merlin.fit.vutbr.cz/kaldi/kaldi_lm.tar.gz || exit 1;
+   fi
+   tar -xvzf kaldi_lm.tar.gz || exit 1;
+   cd kaldi_lm
+   make || exit 1;
+   echo Done making the kaldi_lm tools
+ fi
+) || exit 1;
+
+
+
+if [ ! -f $srcdir/cleaned.gz -o ! -f $srcdir/lexicon.txt ]; then
+  echo "Expecting files $srcdir/cleaned.gz and $srcdir/lexicon.txt to exist";
+  echo "You need to run local/wsj_extend_dict.sh before running this script."
+  exit 1;
+fi
+
+# Get a wordlist-- keep everything but silence, which should not appear in
+# the LM.
+awk '{print $1}' $srcdir/lexicon.txt | grep -v -w '!SIL' > $dir/wordlist.txt
+
+# Get training data with OOV words (w.r.t. our current vocab) replaced with <UNK>.
+echo "Getting training data with OOV words replaced with <UNK> (train_nounk.gz)" 
+gunzip -c $srcdir/cleaned.gz | awk -v w=$dir/wordlist.txt \
+  'BEGIN{while((getline<w)>0) v[$1]=1;}
+  {for (i=1;i<=NF;i++) if ($i in v) printf $i" ";else printf "<UNK> ";print ""}'|sed 's/ $//g' \
+  | gzip -c > $dir/train_nounk.gz
+
+# Get unigram counts (without bos/eos, but this doens't matter here, it's
+# only to get the word-map, which treats them specially & doesn't need their
+# counts).
+# Add a 1-count for each word in word-list by including that in the data,
+# so all words appear.
+gunzip -c $dir/train_nounk.gz | cat - $dir/wordlist.txt | \
+  awk '{ for(x=1;x<=NF;x++) count[$x]++; } END{for(w in count){print count[w], w;}}' | \
+ sort -nr > $dir/unigram.counts
+
+# Get "mapped" words-- a character encoding of the words that makes the common words very short.
+cat $dir/unigram.counts  | awk '{print $2}' | get_word_map.pl "<s>" "</s>" "<UNK>" > $dir/word_map
+
+gunzip -c $dir/train_nounk.gz | awk -v wmap=$dir/word_map 'BEGIN{while((getline<wmap)>0)map[$1]=$2;}
+  { for(n=1;n<=NF;n++) { printf map[$n]; if(n<NF){ printf " "; } else { print ""; }}}' | gzip -c >$dir/train.gz
+
+# To save disk space, remove the un-mapped training data.  We could
+# easily generate it again if needed.
+rm $dir/train_nounk.gz 
+
+train_lm.sh --arpa --lmtype 3gram-mincount $dir
+#Perplexity over 228518.000000 words (excluding 478.000000 OOVs) is 141.444826
+# 7.8 million N-grams.
+
+prune_lm.sh --arpa 6.0 $dir/3gram-mincount/
+# 1.45 million N-grams.
+# Perplexity over 228518.000000 words (excluding 478.000000 OOVs) is 165.394139
+
+train_lm.sh --arpa --lmtype 4gram-mincount $dir
+#Perplexity over 228518.000000 words (excluding 478.000000 OOVs) is 126.734180
+# 10.3 million N-grams.
+
+prune_lm.sh --arpa 7.0 $dir/4gram-mincount
+# 1.50 million N-grams
+# Perplexity over 228518.000000 words (excluding 478.000000 OOVs) is 155.663757
+
+
+exit 0
+
+### Below here, this script is showing various commands that 
+## were run during LM tuning.
+
+train_lm.sh --arpa --lmtype 3gram-mincount $dir
+#Perplexity over 228518.000000 words (excluding 478.000000 OOVs) is 141.444826
+# 7.8 million N-grams.
+
+prune_lm.sh --arpa 3.0 $dir/3gram-mincount/
+#Perplexity over 228518.000000 words (excluding 478.000000 OOVs) is 156.408740
+# 2.5 million N-grams.
+
+prune_lm.sh --arpa 6.0 $dir/3gram-mincount/
+# 1.45 million N-grams.
+# Perplexity over 228518.000000 words (excluding 478.000000 OOVs) is 165.394139
+
+train_lm.sh --arpa --lmtype 4gram-mincount $dir
+#Perplexity over 228518.000000 words (excluding 478.000000 OOVs) is 126.734180
+# 10.3 million N-grams.
+
+prune_lm.sh --arpa 3.0 $dir/4gram-mincount
+#Perplexity over 228518.000000 words (excluding 478.000000 OOVs) is 143.206294
+# 2.6 million N-grams.
+
+prune_lm.sh --arpa 4.0 $dir/4gram-mincount
+# Perplexity over 228518.000000 words (excluding 478.000000 OOVs) is 146.927717
+# 2.15 million N-grams.
+
+prune_lm.sh --arpa 5.0 $dir/4gram-mincount
+# 1.86 million N-grams
+# Perplexity over 228518.000000 words (excluding 478.000000 OOVs) is 150.162023
+
+prune_lm.sh --arpa 7.0 $dir/4gram-mincount
+# 1.50 million N-grams
+# Perplexity over 228518.000000 words (excluding 478.000000 OOVs) is 155.663757
+
+train_lm.sh --arpa --lmtype 3gram $dir
+# Perplexity over 228518.000000 words (excluding 478.000000 OOVs) is 135.692866
+# 20.0 million N-grams
+
+! which ngram-count  \
+  && echo "SRILM tools not installed so not doing the comparison" && exit 1;
+
+#################
+# You could finish the script here if you wanted.
+# Below is to show how to do baselines with SRILM.
+#  You'd have to install the SRILM toolkit first.
+
+heldout_sent=10000 # Don't change this if you want result to be comparable with
+    # kaldi_lm results
+sdir=$dir/srilm # in case we want to use SRILM to double-check perplexities.
+mkdir -p $sdir
+gunzip -c $srcdir/cleaned.gz | head -$heldout_sent > $sdir/cleaned.heldout
+gunzip -c $srcdir/cleaned.gz | tail -n +$heldout_sent > $sdir/cleaned.train
+(echo "<s>"; echo "</s>" ) | cat - $dir/wordlist.txt > $sdir/wordlist.final.s
+
+# 3-gram:
+ngram-count -text $sdir/cleaned.train -order 3 -limit-vocab -vocab $sdir/wordlist.final.s -unk \
+  -map-unk "<UNK>" -kndiscount -interpolate -lm $sdir/srilm.o3g.kn.gz
+ngram -lm $sdir/srilm.o3g.kn.gz -ppl $sdir/cleaned.heldout # consider -debug 2
+#file data/local/local_lm/srilm/cleaned.heldout: 10000 sentences, 218996 words, 478 OOVs
+#0 zeroprobs, logprob= -491456 ppl= 141.457 ppl1= 177.437
+
+# Trying 4-gram:
+ngram-count -text $sdir/cleaned.train -order 4 -limit-vocab -vocab $sdir/wordlist.final.s -unk \
+  -map-unk "<UNK>" -kndiscount -interpolate -lm $sdir/srilm.o4g.kn.gz
+ngram -order 4 -lm $sdir/srilm.o4g.kn.gz -ppl $sdir/cleaned.heldout 
+#file data/local/local_lm/srilm/cleaned.heldout: 10000 sentences, 218996 words, 478 OOVs
+#0 zeroprobs, logprob= -480939 ppl= 127.233 ppl1= 158.822
+
+#3-gram with pruning:
+ngram-count -text $sdir/cleaned.train -order 3 -limit-vocab -vocab $sdir/wordlist.final.s -unk \
+  -prune 0.0000001 -map-unk "<UNK>" -kndiscount -interpolate -lm $sdir/srilm.o3g.pr7.kn.gz
+ngram -lm $sdir/srilm.o3g.pr7.kn.gz -ppl $sdir/cleaned.heldout 
+#file data/local/local_lm/srilm/cleaned.heldout: 10000 sentences, 218996 words, 478 OOVs
+#0 zeroprobs, logprob= -510828 ppl= 171.947 ppl1= 217.616
+# Around 2.25M N-grams.
+# Note: this is closest to the experiment done with "prune_lm.sh --arpa 3.0 $dir/3gram-mincount/"
+# above, which gave 2.5 million N-grams and a perplexity of 156.
+
+# Note: all SRILM experiments above fully discount all singleton 3 and 4-grams.
+# You can use -gt3min=0 and -gt4min=0 to stop this (this will be comparable to
+# the kaldi_lm experiments above without "-mincount".
+
+##  From here is how to train with
+# IRSTLM.  This is not really working at the moment.
+export IRSTLM=$KALDI_ROOT/tools/irstlm/
+
+idir=$dir/irstlm
+mkdir $idir
+gunzip -c $srcdir/cleaned.gz | tail -n +$heldout_sent | $IRSTLM/scripts/add-start-end.sh | \
+  gzip -c > $idir/train.gz
+
+$IRSTLM/bin/dict -i=WSJ.cleaned.irstlm.txt -o=dico -f=y -sort=no
+ cat dico | gawk 'BEGIN{while (getline<"vocab.20k.nooov") v[$1]=1; print "DICTIONARY 0 "length(v);}FNR>1{if ($1 in v)\
+{print $0;}}' > vocab.irstlm.20k
+
+
+$IRSTLM/bin/build-lm.sh -i "gunzip -c $idir/train.gz" -o $idir/lm_3gram.gz  -p yes \
+  -n 3 -s improved-kneser-ney -b yes
+# Testing perplexity with SRILM tools:
+ngram -lm $idir/lm_3gram.gz  -ppl $sdir/cleaned.heldout 
+#data/local/local_lm/irstlm/lm_3gram.gz: line 162049: warning: non-zero probability for <unk> in closed-vocabulary LM
+#file data/local/local_lm/srilm/cleaned.heldout: 10000 sentences, 218996 words, 0 OOVs
+#0 zeroprobs, logprob= -513670 ppl= 175.041 ppl1= 221.599
+
+# Perplexity is very bad (should be ~141, since we used -p option,
+# not 175),
+# but adding -debug 3 to the command line shows that
+# the IRSTLM LM does not seem to sum to one properly, so it seems that
+# it produces an LM that isn't interpretable in the normal way as an ARPA
+# LM.
+
+
+
--- a/egs/aurora4/s5/local/wsj_train_rnnlms.sh
+++ b/egs/aurora4/s5/local/wsj_train_rnnlms.sh
@ -0,0 +1,153 @@
+#!/bin/bash 
+
+# Copyright 2012  Johns Hopkins University (author: Daniel Povey)  Tony Robinson
+
+# This script trains LMs on the WSJ LM-training data.
+# It requires that you have already run wsj_extend_dict.sh,
+# to get the larger-size dictionary including all of CMUdict
+# plus any OOVs and possible acronyms that we could easily 
+# derive pronunciations for.
+
+# This script takes no command-line arguments but takes the --cmd option.
+
+# Begin configuration section.
+rand_seed=0
+cmd=run.pl
+nwords=10000 # This is how many words we're putting in the vocab of the RNNLM. 
+hidden=30
+class=200 # Num-classes... should be somewhat larger than sqrt of nwords.
+direct=1000 # Probably number of megabytes to allocate for hash-table for "direct" connections.
+rnnlm_ver=rnnlm-0.3e # version of RNNLM to use
+# End configuration section.
+
+[ -f ./path.sh ] && . ./path.sh
+. utils/parse_options.sh
+
+if [ $# != 1 ]; then
+   echo "Usage: local/wsj_train_rnnlms.sh [options] <dest-dir>"
+   echo "For options, see top of script file"
+   exit 1;
+fi
+
+dir=$1
+srcdir=data/local/dict_larger
+mkdir -p $dir
+
+export PATH=$KALDI_ROOT/tools/$rnnlm_ver:$PATH
+
+
+( # First make sure the kaldi_lm toolkit is installed.
+ # Note: this didn't work out of the box for me, I had to
+ # change the g++ version to just "g++" (no cross-compilation
+ # needed for me as I ran on a machine that had been setup
+ # as 64 bit by default.
+ cd $KALDI_ROOT/tools || exit 1;
+ if [ -d $rnnlm_ver ]; then
+   echo Not installing the rnnlm toolkit since it is already there.
+ else
+   echo Downloading and installing the rnnlm tools
+   # http://www.fit.vutbr.cz/~imikolov/rnnlm/$rnnlm_ver.tgz
+   if [ ! -f $rnnlm_ver.tgz ]; then
+     wget http://www.fit.vutbr.cz/~imikolov/rnnlm/$rnnlm_ver.tgz || exit 1;
+   fi
+   mkdir $rnnlm_ver
+   cd $rnnlm_ver
+   tar -xvzf ../$rnnlm_ver.tgz || exit 1;
+   make CC=g++ || exit 1;
+   echo Done making the rnnlm tools
+ fi
+) || exit 1;
+
+
+if [ ! -f $srcdir/cleaned.gz -o ! -f $srcdir/lexicon.txt ]; then
+  echo "Expecting files $srcdir/cleaned.gz and $srcdir/wordlist.final to exist";
+  echo "You need to run local/wsj_extend_dict.sh before running this script."
+  exit 1;
+fi
+
+cat $srcdir/lexicon.txt | awk '{print $1}' | grep -v -w '!SIL' > $dir/wordlist.all
+
+# Get training data with OOV words (w.r.t. our current vocab) replaced with <UNK>.
+echo "Getting training data with OOV words replaced with <UNK> (train_nounk.gz)" 
+gunzip -c $srcdir/cleaned.gz | awk -v w=$dir/wordlist.all \
+  'BEGIN{while((getline<w)>0) v[$1]=1;}
+  {for (i=1;i<=NF;i++) if ($i in v) printf $i" ";else printf "<UNK> ";print ""}'|sed 's/ $//g' \
+  | gzip -c > $dir/all.gz
+
+echo "Splitting data into train and validation sets."
+heldout_sent=10000
+gunzip -c $dir/all.gz | head -n $heldout_sent > $dir/valid.in # validation data
+gunzip -c $dir/all.gz | tail -n +$heldout_sent | \
+ perl -e ' use List::Util qw(shuffle); @A=<>; print join("", shuffle(@A)); ' \
+  > $dir/train.in # training data
+
+
+  # The rest will consist of a word-class represented by <RNN_UNK>, that
+  # maps (with probabilities) to a whole class of words.
+
+# Get unigram counts from our training data, and use this to select word-list
+# for RNNLM training; e.g. 10k most frequent words.  Rest will go in a class
+# that we (manually, at the shell level) assign probabilities for words that
+# are in that class.  Note: this word-list doesn't need to include </s>; this
+# automatically gets added inside the rnnlm program.
+# Note: by concatenating with $dir/wordlist.all, we are doing add-one
+# smoothing of the counts.
+
+cat $dir/train.in $dir/wordlist.all | grep -v '</s>' | grep -v '<s>' | \
+  awk '{ for(x=1;x<=NF;x++) count[$x]++; } END{for(w in count){print count[w], w;}}' | \
+  sort -nr > $dir/unigram.counts
+
+head -$nwords $dir/unigram.counts | awk '{print $2}' > $dir/wordlist.rnn
+
+tail -n +$nwords $dir/unigram.counts > $dir/unk_class.counts
+
+tot=`awk '{x=x+$1} END{print x}' $dir/unk_class.counts`
+awk -v tot=$tot '{print $2, ($1*1.0/tot);}' <$dir/unk_class.counts  >$dir/unk.probs
+
+
+for type in train valid; do
+  cat $dir/$type.in | awk -v w=$dir/wordlist.rnn \
+    'BEGIN{while((getline<w)>0) v[$1]=1;}
+    {for (i=1;i<=NF;i++) if ($i in v) printf $i" ";else printf "<RNN_UNK> ";print ""}'|sed 's/ $//g' \
+    > $dir/$type
+done
+rm $dir/train.in # no longer needed-- and big.
+
+# Now randomize the order of the training data.
+cat $dir/train | awk -v rand_seed=$rand_seed 'BEGIN{srand(rand_seed);} {printf("%f\t%s\n", rand(), $0);}' | \
+ sort | cut -f 2 > $dir/foo
+mv $dir/foo $dir/train
+
+# OK we'll train the RNNLM on this data.
+
+# todo: change 100 to 320.
+# using 100 classes as square root of 10k.
+echo "Training RNNLM (note: this uses a lot of memory! Run it on a big machine.)"
+#time rnnlm -train $dir/train -valid $dir/valid -rnnlm $dir/100.rnnlm \
+#  -hidden 100 -rand-seed 1 -debug 2 -class 100 -bptt 2 -bptt-block 20 \
+#  -direct-order 4 -direct 1000 -binary >& $dir/rnnlm1.log &
+
+$cmd $dir/rnnlm.log \
+   $KALDI_ROOT/tools/$rnnlm_ver/rnnlm -independent -train $dir/train -valid $dir/valid \
+   -rnnlm $dir/rnnlm -hidden $hidden -rand-seed 1 -debug 2 -class $class -bptt 2 -bptt-block 20 \
+   -direct-order 4 -direct $direct -binary || exit 1;
+
+
+# make it like a Kaldi table format, with fake utterance-ids.
+cat $dir/valid.in | awk '{ printf("uttid-%d ", NR); print; }' > $dir/valid.with_ids
+
+utils/rnnlm_compute_scores.sh $dir $dir/tmp.valid $dir/valid.with_ids \
+  $dir/valid.scores
+nw=`wc -w < $dir/valid.with_ids` # Note: valid.with_ids includes utterance-ids which
+  # is one per word, to account for the </s> at the end of each sentence; this is the
+  # correct number to normalize buy.
+p=`awk -v nw=$nw '{x=x+$2} END{print exp(x/nw);}' <$dir/valid.scores` 
+echo Perplexity is $p | tee $dir/perplexity.log
+
+rm $dir/train $dir/all.gz
+
+# This is a better setup, but takes a long time to train:
+#echo "Training RNNLM (note: this uses a lot of memory! Run it on a big machine.)"
+#time rnnlm -train $dir/train -valid $dir/valid -rnnlm $dir/320.rnnlm \
+#  -hidden 320 -rand-seed 1 -debug 2 -class 300 -bptt 2 -bptt-block 20 \
+#  -direct-order 4 -direct 2000 -binary
--- a/egs/aurora4/s5/path.sh
+++ b/egs/aurora4/s5/path.sh
@ -0,0 +1,3 @@
+export KALDI_ROOT=`pwd`/../../..
+export PATH=$PWD/utils/:$KALDI_ROOT/src/bin:$KALDI_ROOT/tools/openfst/bin:$KALDI_ROOT/src/fstbin/:$KALDI_ROOT/src/gmmbin/:$KALDI_ROOT/src/featbin/:$KALDI_ROOT/src/lm/:$KALDI_ROOT/src/sgmmbin/:$KALDI_ROOT/src/sgmm2bin/:$KALDI_ROOT/src/fgmmbin/:$KALDI_ROOT/src/latbin/:$KALDI_ROOT/src/nnetbin:$KALDI_ROOT/src/nnet-cpubin/:$KALDI_ROOT/src/kwsbin:$PWD:$PATH
+export LC_ALL=C
--- a/egs/aurora4/s5/run.sh
+++ b/egs/aurora4/s5/run.sh
@ -0,0 +1,152 @@
+#!/bin/bash
+
+. ./cmd.sh ## You'll want to change cmd.sh to something that will work on your system.
+           ## This relates to the queue.
+
+# This is a shell script, but it's recommended that you run the commands one by
+# one by copying and pasting into the shell.
+
+case 0 in    #goto here
+    1)
+
+
+aurora4=/mnt/spdb/aurora4
+#we need lm, trans, from WSJ0 CORPUS
+wsj0=/mnt/spdb/wall_street_journal
+
+local/aurora4_data_prep.sh $aurora4 $wsj0
+
+local/wsj_prepare_dict.sh || exit 1;
+
+utils/prepare_lang.sh data/local/dict "<SPOKEN_NOISE>" data/local/lang_tmp data/lang || exit 1;
+
+local/aurora4_format_data.sh || exit 1;
+
+# Now make MFCC features.
+# mfccdir should be some place with a largish disk where you
+# want to store MFCC features.
+mfccdir=mfcc
+for x in train_si84_clean train_si84_multi test_eval92 test_0166 dev_0330 dev_1206; do 
+ steps/make_mfcc.sh  --nj 10 \
+   data/$x exp/make_mfcc/$x $mfccdir || exit 1;
+ steps/compute_cmvn_stats.sh data/$x exp/make_mfcc/$x $mfccdir || exit 1;
+done
+
+# make fbank features
+fbankdir=fbank
+mkdir -p data-fbank
+for x in train_si84_clean train_si84_multi dev_0330 dev_1206 test_eval92 test_0166; do
+  cp -r data/$x data-fbank/$x
+  steps/make_fbank.sh --nj 10 \
+    data-fbank/$x exp/make_fbank/$x $fbankdir || exit 1;
+done
+
+# Note: the --boost-silence option should probably be omitted by default
+# for normal setups.  It doesn't always help. [it's to discourage non-silence
+# models from modeling silence.]
+#steps/train_mono.sh --boost-silence 1.25 --nj 10  \
+#  data/train_si84_clean data/lang exp/mono0a || exit 1;
+
+steps/train_mono.sh --boost-silence 1.25 --nj 10  \
+  data/train_si84_multi data/lang exp/mono0a_multi || exit 1;
+#(
+# utils/mkgraph.sh --mono data/lang_test_tgpr exp/mono0a exp/mono0a/graph_tgpr && \
+# steps/decode.sh --nj 8  \
+#   exp/mono0a/graph_tgpr data/test_eval92 exp/mono0a/decode_tgpr_eval92 
+#) &
+
+#steps/align_si.sh --boost-silence 1.25 --nj 10  \
+#   data/train_si84_clean data/lang exp/mono0a exp/mono0a_ali || exit 1;
+
+steps/align_si.sh --boost-silence 1.25 --nj 10  \
+   data/train_si84_multi data/lang exp/mono0a_multi exp/mono0a_multi_ali || exit 1;
+
+#steps/train_deltas.sh --boost-silence 1.25 \
+#    2000 10000 data/train_si84_clean data/lang exp/mono0a_ali exp/tri1 || exit 1;
+
+steps/train_deltas.sh --boost-silence 1.25 \
+    2000 10000 data/train_si84_multi data/lang exp/mono0a_multi_ali exp/tri1_multi || exit 1;
+
+
+while [ ! -f data/lang_test_tgpr/tmp/LG.fst ] || \
+   [ -z data/lang_test_tgpr/tmp/LG.fst ]; do
+  sleep 20;
+done
+sleep 30;
+# or the mono mkgraph.sh might be writing 
+# data/lang_test_tgpr/tmp/LG.fst which will cause this to fail.
+
+steps/align_si.sh --nj 10 \
+  data/train_si84_multi data/lang exp/tri1_multi exp/tri1_multi_ali_si84 || exit 1;
+
+steps/train_deltas.sh  \
+  2500 15000 data/train_si84_multi data/lang exp/tri1_multi_ali_si84 exp/tri2a_multi || exit 1;
+
+
+steps/train_lda_mllt.sh \
+   --splice-opts "--left-context=3 --right-context=3" \
+   2500 15000 data/train_si84_multi data/lang exp/tri1_multi_ali_si84 exp/tri2b_multi || exit 1;
+
+
+utils/mkgraph.sh data/lang_test_tgpr_5k exp/tri2b_multi exp/tri2b_multi/graph_tgpr_5k || exit 1;
+steps/decode.sh --nj 8 \
+  exp/tri2b_multi/graph_tgpr_5k data/test_eval92 exp/tri2b_multi/decode_tgpr_5k_eval92 || exit 1;
+
+# Align tri2b system with si84 multi-condition data.
+steps/align_si.sh  --nj 10 \
+  --use-graphs true data/train_si84_multi data/lang exp/tri2b_multi exp/tri2b_multi_ali_si84  || exit 1;
+
+steps/align_si.sh  --nj 10 \
+  data/dev_0330 data/lang exp/tri2b_multi exp/tri2b_multi_ali_dev_0330 || exit 1;
+
+steps/align_si.sh  --nj 10 \
+  data/dev_1206 data/lang exp/tri2b_multi exp/tri2b_multi_ali_dev_1206 || exit 1;
+
+#Now begin train DNN systems on multi data
+. ./path.sh
+#RBM pretrain
+dir=exp/tri3a_dnn_pretrain
+$cuda_cmd $dir/_pretrain_dbn.log \
+  steps/pretrain_dbn.sh --use-gpu-id 0 --nn-depth 7 --rbm-iter 3 data-fbank/train_si84_multi $dir
+
+dir=exp/tri3a_dnn
+ali=exp/tri2b_multi_ali_si84
+ali_dev=exp/tri2b_multi_ali_dev_0330
+feature_transform=exp/tri3a_dnn_pretrain/final.feature_transform
+dbn=exp/tri3a_dnn_pretrain/7.dbn
+$cuda_cmd $dir/_train_nnet.log \
+  steps/train_nnet.sh --feature-transform $feature_transform --dbn $dbn --hid-layers 0 --learn-rate 0.008 --use-gpu-id 0 \
+  data-fbank/train_si84_multi data-fbank/dev_0330 data/lang $ali $ali_dev $dir || exit 1;
+
+utils/mkgraph.sh data/lang_test_tgpr_5k exp/tri3a_dnn exp/tri3a_dnn/graph_tgpr_5k || exit 1;
+dir=exp/tri3a_dnn
+steps/decode_nnet.sh --nj 8 --acwt 0.10 --config conf/decode_dnn.config \
+  exp/tri3a_dnn/graph_tgpr_5k data-fbank/test_eval92 $dir/decode_tgpr_5k_eval92 || exit 1;
+
+
+#realignments
+srcdir=exp/tri3a_dnn
+steps/align_nnet.sh --nj 10 \
+  data-fbank/train_si84_multi data/lang $srcdir ${srcdir}_ali_si84_multi || exit 1;
+steps/align_nnet.sh --nj 10 \
+  data-fbank/dev_0330 data/lang $srcdir ${srcdir}_ali_dev_0330 || exit 1;
+
+#train system again 
+
+dir=exp/tri4a_dnn
+ali=exp/tri3a_dnn_ali_si84_multi
+ali_dev=exp/tri3a_dnn_ali_dev_0330
+feature_transform=exp/tri3a_dnn_pretrain/final.feature_transform
+dbn=exp/tri3a_dnn_pretrain/7.dbn
+$cuda_cmd $dir/_train_nnet.log \
+  steps/train_nnet.sh --feature-transform $feature_transform --dbn $dbn --hid-layers 0 --learn-rate 0.008 --use-gpu-id 0 \
+  data-fbank/train_si84_multi data-fbank/dev_0330 data/lang $ali $ali_dev $dir || exit 1;
+
+utils/mkgraph.sh data/lang_test_tgpr_5k exp/tri4a_dnn exp/tri4a_dnn/graph_tgpr_5k || exit 1;
+dir=exp/tri4a_dnn
+steps/decode_nnet.sh --nj 8 --acwt 0.10 --config conf/decode_dnn.config \
+  exp/tri4a_dnn/graph_tgpr_5k data-fbank/test_eval92 $dir/decode_tgpr_5k_eval92 || exit 1;
+
+
+# DNN Sequential DT training
+#......
--- a/egs/aurora4/s5/steps
+++ b/egs/aurora4/s5/steps
@ -0,0 +1 @@
+../../wsj/s5/steps
--- a/egs/aurora4/s5/utils
+++ b/egs/aurora4/s5/utils
@ -0,0 +1 @@
+../../wsj/s5/utils
				`@ -0,0 +1 @@`
				`--use-energy=false # only non-default option.`