Merge branch 'reformat-code'

marytts-builder/src/main/java/marytts/tools/analysis/DrawHistogram.java

@@ -203,7 +203,7 @@ public class DrawHistogram extends javax.swing.JFrame {
 	}// </editor-fold>//GEN-END:initComponents
 
 	private void changeBandWidth(javax.swing.event.ChangeEvent evt) {// GEN-FIRST:event_changeBandWidth
-	// TODO add your handling code here:
+		// TODO add your handling code here:
 		try {
 			int bwSlide = bandwidth.getValue();
 			this.jLabel2.setText(Integer.toString(bwSlide));

marytts-builder/src/main/java/marytts/tools/dbselection/DatabaseCFProvider.java

@@ -27,13 +27,15 @@ public class DatabaseCFProvider implements CoverageFeatureProvider {
 
 	private DBHandler dbHandler;
 	private int[] sentenceIDs;
-	
+
 	public DatabaseCFProvider(DBHandler dbHandler, String condition) {
 		this.dbHandler = dbHandler;
 		this.sentenceIDs = dbHandler.getIdListOfType("dbselection", condition);
 	}
-	
-	/* (non-Javadoc)
+
+	/*
+	 * (non-Javadoc)
+	 * 
 	 * @see marytts.tools.dbselection.CoverageFeatureProvider#getCoverageFeatures(int)
 	 */
 	@Override
@@ -41,26 +43,32 @@ public class DatabaseCFProvider implements CoverageFeatureProvider {
 		return dbHandler.getFeatures(sentenceIDs[i]);
 	}
 
-	/* (non-Javadoc)
+	/*
+	 * (non-Javadoc)
+	 * 
 	 * @see marytts.tools.dbselection.CoverageFeatureProvider#getNumSentences()
 	 */
 	@Override
 	public int getNumSentences() {
 		return sentenceIDs.length;
 	}
-	
+
 	@Override
 	public int getID(int i) {
 		return sentenceIDs[i];
 	}
 
 	/**
-	 * Get in-memory access to a subset of the features. The features will be
-	 * accessible from the in-memory provider with index numbers 0..(len-1). 
-	 * @param off first index of features to be made available in the in-memory provider.
-	 * @param len number of features to provide.
+	 * Get in-memory access to a subset of the features. The features will be accessible from the in-memory provider with index
+	 * numbers 0..(len-1).
+	 * 
+	 * @param off
+	 *            first index of features to be made available in the in-memory provider.
+	 * @param len
+	 *            number of features to provide.
 	 * @return an in-memory context feature provider.
-	 * @throws IndexOutOfBoundsException if off or off+len-1 are outside the range from 0 to getNumSentences()-1.
+	 * @throws IndexOutOfBoundsException
+	 *             if off or off+len-1 are outside the range from 0 to getNumSentences()-1.
 	 */
 	public InMemoryCFProvider getFeaturesInMemory(int off, int len) {
 		int[] ids = new int[len];

marytts-builder/src/main/java/marytts/tools/dbselection/InMemoryCFProvider.java

@@ -27,13 +27,18 @@ public class InMemoryCFProvider implements CoverageFeatureProvider {
 
 	private byte[][] data;
 	private int[] ids;
-	
+
 	/**
 	 * Instantiate an in-memory coverage feature provider
-	 * @param data the data to provide.
-	 * @param ids optionally, an array of unique id numbers. if this is null, the index number is used as the id number.
-	 * @throws NullPointerException if data is null
-	 * @throws IllegalArgumentException if ids is given but has different length than data.
+	 * 
+	 * @param data
+	 *            the data to provide.
+	 * @param ids
+	 *            optionally, an array of unique id numbers. if this is null, the index number is used as the id number.
+	 * @throws NullPointerException
+	 *             if data is null
+	 * @throws IllegalArgumentException
+	 *             if ids is given but has different length than data.
 	 */
 	public InMemoryCFProvider(byte[][] data, int[] ids) {
 		if (data == null) {
@@ -45,8 +50,10 @@ public class InMemoryCFProvider implements CoverageFeatureProvider {
 			throw new IllegalArgumentException("ID array does not have same length as data vector");
 		}
 	}
-	
-	/* (non-Javadoc)
+
+	/*
+	 * (non-Javadoc)
+	 * 
 	 * @see marytts.tools.dbselection.CoverageFeatureProvider#getCoverageFeatures(int)
 	 */
 	@Override
@@ -54,14 +61,16 @@ public class InMemoryCFProvider implements CoverageFeatureProvider {
 		return data[i];
 	}
 
-	/* (non-Javadoc)
+	/*
+	 * (non-Javadoc)
+	 * 
 	 * @see marytts.tools.dbselection.CoverageFeatureProvider#getNumSentences()
 	 */
 	@Override
 	public int getNumSentences() {
 		return data.length;
 	}
-	
+
 	@Override
 	public int getID(int i) {
 		if (ids != null) {

marytts-builder/src/main/java/marytts/tools/dbselection/SelectionFunction.java

@@ -52,227 +52,221 @@ import org.w3c.dom.traversal.TreeWalker;
 import org.xml.sax.SAXException;
 
 /**
- * Selects sentences from a given set using the greedy algorithm.
- * At each step, the most useful sentence is added to the set of
- * selected sentences.
- * Selection is stopped when the stop criterion is reached.
- * Usefulness of sentences is determined by CoverageDefinition.
+ * Selects sentences from a given set using the greedy algorithm. At each step, the most useful sentence is added to the set of
+ * selected sentences. Selection is stopped when the stop criterion is reached. Usefulness of sentences is determined by
+ * CoverageDefinition.
  * 
  * @author Anna Hunecke
  *
  */
-public class SelectionFunction{
+public class SelectionFunction {
 
-    //maximum number of sentences to select
-    private int maxNumSents;
-    //the vectors that are selected next 
-    private byte[] selectedVectors;
-    //the filename of the sentence that is selected next
-    //private String selectedBasename;
-    private int selectedIdSentence;
-    // the usefulness of the selected sentence
-    private double selectedUsefulness;
-    
-    //if true, algorithm stop after maxNumSents are selected
-    private boolean stopNumSentences;
-    //if true, algorithm stops when maximum coverage of 
-    //simple diphones is reached
-    private boolean stopSimpleDiphones;
-    //if true, algorithm stops when maximum coverage of 
-    //clustered diphones is reached
-    private boolean stopClusteredDiphones;
-    //if true, algorithm stops when maximum coverage of 
-    //simple prosody is reached
-    private boolean stopSimpleProsody;
-    //if true, algorithm stops when maximum coverage of 
-    //clustered prosody is reached
-    private boolean stopClusteredProsody;
-    //if true, print information to command line
-    private boolean verbose;
+	// maximum number of sentences to select
+	private int maxNumSents;
+	// the vectors that are selected next
+	private byte[] selectedVectors;
+	// the filename of the sentence that is selected next
+	// private String selectedBasename;
+	private int selectedIdSentence;
+	// the usefulness of the selected sentence
+	private double selectedUsefulness;
 
-    /**
-     * Build a new Selection Function
-     * 
-     */
-    public SelectionFunction(){
-        }
+	// if true, algorithm stop after maxNumSents are selected
+	private boolean stopNumSentences;
+	// if true, algorithm stops when maximum coverage of
+	// simple diphones is reached
+	private boolean stopSimpleDiphones;
+	// if true, algorithm stops when maximum coverage of
+	// clustered diphones is reached
+	private boolean stopClusteredDiphones;
+	// if true, algorithm stops when maximum coverage of
+	// simple prosody is reached
+	private boolean stopSimpleProsody;
+	// if true, algorithm stops when maximum coverage of
+	// clustered prosody is reached
+	private boolean stopClusteredProsody;
+	// if true, print information to command line
+	private boolean verbose;
 
-    /**
-     * Check, if given stop criterion is okay.
-     * At the same time, initialise stop criterion
-     * as this SelectionFunction's stop criterion
-     * 
-     * @param stopString the stop criterion
-     * @return true if stopString can be parsed, false otherwise
-     */
-    public boolean stopIsOkay(String stopString){
-        //set all stop criteria to false
-        stopNumSentences = false;
-        stopSimpleDiphones = false;
-        stopClusteredDiphones = false;
-        stopSimpleProsody = false;
-        stopClusteredProsody = false;
-        //split the stopString
-        System.out.println("\nChecking stop criterion:");
-        String[] split = stopString.split(" ");
-        int i=0;
-        while (split.length > i){
-            if (split[i].startsWith("numSentences")){
-                //criterion is numSentences
-                stopNumSentences = true;
-                if (split.length > i+1){
-                    //read in the maximum number of sentences
-                    maxNumSents = Integer.parseInt(split[++i]);
-                } else {
-                    //maximum number of sentences is missing - can not parse
-                    return false;
-                }
-                System.out.println("Stop: num sentences "+maxNumSents);
-            } else {
-                if (split[i].equals("simpleDiphones")){
-                    //stop criterion is simpleDiphones
-                    stopSimpleDiphones = true;
-                    System.out.println("Stop: simpleDiphones");
-                } else {
-                    if (split[i].equals("clusteredDiphones")){
-                        //stop criterion is clusteredDiphones
-                        stopClusteredDiphones = true;
-                        System.out.println("Stop: clusteredDiphones");
-                    } else {
-                        if (split[i].equals("simpleProsody")){
-                            //stop criterion is simpleProsody
-                            stopSimpleProsody = true;
-                            System.out.println("Stop: simpleProsody");
-                        } else {
-                            if (split[i].equals("clusteredProsody")){
-                                //stop criterion is clusteredProsody
-                                stopClusteredProsody = true;
-                                System.out.println("Stop: clusteredProsody");
-                            } else {
-                                //unknown stop criterion - can not parse
-                                return false;
-                            }
-                        }
-                    }
-                }
-            }
-            i++;
-        }
-        //everything allright
-        return true;
-    }
+	/**
+	 * Build a new Selection Function
+	 * 
+	 */
+	public SelectionFunction() {
+	}
 
-   /**
-    * Select a set of vectors according to their usefulness
-    * which is defined by the coverageDefinition. Stop, when 
-    * the stop criterion is reached
-    * 
-    * @param selectedIdSents the list of selected id sentences
-    * @param unwantedIdSents the list of unwanted id sentences
-    * @param coverageDefinition the coverage definition for the feature vectors
-    * @param logFile the logFile to document the progress
-    * @param basenameList the list of filenames of the sentences
-    * @param holdVectorsInMemory if true, get vectors from coverage definition,
-    *                            if false, read vectors from disk 
-    * @param verbose print output also to command line
-    * @return the list of selected filenames
- * @throws Exception 
-    */ 
-    public void select(Set<Integer>selectedIdSents,
-                Set<Integer>unwantedIdSents,
-                CoverageDefinition coverageDefinition,
-                PrintWriter logFile,
-                CoverageFeatureProvider cfProvider,
-                boolean verboseSelect,
-                DBHandler wikiToDB) //throws IOException
-                throws Exception {
-        this.verbose = verboseSelect;
-        int sentIndex = selectedIdSents.size()+1;
-        selectedVectors = null;  
-        DateFormat fullDate = new SimpleDateFormat("HH_mm_ss");
-       
-        // create the selectedSentences table 
-        // while the stop criterion is not reached      
-        while(!stopCriterionIsReached(selectedIdSents, coverageDefinition)){ 
-            
-            //select the next sentence  
-            //selectNext(coverageDefinition, logFile, sentIndex, basenameList, vectorArray);
-            boolean haveSelected = selectNext(selectedIdSents, unwantedIdSents, coverageDefinition, cfProvider);
+	/**
+	 * Check, if given stop criterion is okay. At the same time, initialise stop criterion as this SelectionFunction's stop
+	 * criterion
+	 * 
+	 * @param stopString
+	 *            the stop criterion
+	 * @return true if stopString can be parsed, false otherwise
+	 */
+	public boolean stopIsOkay(String stopString) {
+		// set all stop criteria to false
+		stopNumSentences = false;
+		stopSimpleDiphones = false;
+		stopClusteredDiphones = false;
+		stopSimpleProsody = false;
+		stopClusteredProsody = false;
+		// split the stopString
+		System.out.println("\nChecking stop criterion:");
+		String[] split = stopString.split(" ");
+		int i = 0;
+		while (split.length > i) {
+			if (split[i].startsWith("numSentences")) {
+				// criterion is numSentences
+				stopNumSentences = true;
+				if (split.length > i + 1) {
+					// read in the maximum number of sentences
+					maxNumSents = Integer.parseInt(split[++i]);
+				} else {
+					// maximum number of sentences is missing - can not parse
+					return false;
+				}
+				System.out.println("Stop: num sentences " + maxNumSents);
+			} else {
+				if (split[i].equals("simpleDiphones")) {
+					// stop criterion is simpleDiphones
+					stopSimpleDiphones = true;
+					System.out.println("Stop: simpleDiphones");
+				} else {
+					if (split[i].equals("clusteredDiphones")) {
+						// stop criterion is clusteredDiphones
+						stopClusteredDiphones = true;
+						System.out.println("Stop: clusteredDiphones");
+					} else {
+						if (split[i].equals("simpleProsody")) {
+							// stop criterion is simpleProsody
+							stopSimpleProsody = true;
+							System.out.println("Stop: simpleProsody");
+						} else {
+							if (split[i].equals("clusteredProsody")) {
+								// stop criterion is clusteredProsody
+								stopClusteredProsody = true;
+								System.out.println("Stop: clusteredProsody");
+							} else {
+								// unknown stop criterion - can not parse
+								return false;
+							}
+						}
+					}
+				}
+			}
+			i++;
+		}
+		// everything allright
+		return true;
+	}
 
-            if (haveSelected) {
-            	assert selectedIdSentence >= 0;
-                selectedIdSents.add(selectedIdSentence);
-                
-                //print information
-                String msg = "Sentence "+sentIndex+" ("+selectedIdSentence+"), score: "+selectedUsefulness;
-                if (verbose) {
-                  System.out.println(msg);  
-                }
-                logFile.println(msg);
-                logFile.flush();
-            } else {
-                //nothing more to select
-                //System.out.println("Nothing more to select");
-                logFile.println("Nothing more to select");
-                break;
-            }
-            // the selected sentences will be marked as selected=true in the DB
-            Date date = new Date();
-            System.out.println("  " + sentIndex + " selectedId=" + selectedIdSentence + "  " 
-                    + fullDate.format(date)); 
-            // Mark the sentence as selected in dbselection
-            wikiToDB.setSentenceRecord(selectedIdSentence, "selected", true);
-            // Insert selected sentence in table
-            wikiToDB.insertSelectedSentence(selectedIdSentence, false);
+	/**
+	 * Select a set of vectors according to their usefulness which is defined by the coverageDefinition. Stop, when the stop
+	 * criterion is reached
+	 * 
+	 * @param selectedIdSents
+	 *            the list of selected id sentences
+	 * @param unwantedIdSents
+	 *            the list of unwanted id sentences
+	 * @param coverageDefinition
+	 *            the coverage definition for the feature vectors
+	 * @param logFile
+	 *            the logFile to document the progress
+	 * @param basenameList
+	 *            the list of filenames of the sentences
+	 * @param holdVectorsInMemory
+	 *            if true, get vectors from coverage definition, if false, read vectors from disk
+	 * @param verbose
+	 *            print output also to command line
+	 * @return the list of selected filenames
+	 * @throws Exception
+	 */
+	public void select(Set<Integer> selectedIdSents, Set<Integer> unwantedIdSents, CoverageDefinition coverageDefinition,
+			PrintWriter logFile, CoverageFeatureProvider cfProvider, boolean verboseSelect, DBHandler wikiToDB) // throws
+																												// IOException
+			throws Exception {
+		this.verbose = verboseSelect;
+		int sentIndex = selectedIdSents.size() + 1;
+		selectedVectors = null;
+		DateFormat fullDate = new SimpleDateFormat("HH_mm_ss");
 
-            //add the selected sentence to the set
-            //selectedFilenames.add(selectedBasename);
-            // selectedIdSents.add(selectedIdSentence); already done in selectNext
-            //update coverageDefinition         
-            coverageDefinition.updateCover(selectedVectors);
-            sentIndex++;
-        }  
-        //print out total number of sentences
-        sentIndex--;
-        System.out.println("Total number of selected sentences in TABLE: " + wikiToDB.getSelectedSentencesTableName()
-                + " = " + sentIndex);
-        
-        
-        int sel[] = wikiToDB.getIdListOfType("dbselection", "selected=true and unwanted=false");
-        
-        if( sel != null){
-          // saving sentences in a file
-          System.out.println("Saving selected sentences in ./selected.log");
-          PrintWriter selectedLog = new PrintWriter(new FileWriter(new File("./selected.log")));
-          
-          System.out.println("Saving selected sentences and transcriptions in ./selected_text_transcription.log");
-          PrintWriter selected_tra_Log = new PrintWriter(new FileWriter(new File("./selected_text_transcription.log")));
-          
-            
-          String str;
-          for(int i=0; i<sel.length; i++){
-            // not sure if we need to make another table???
-            // str = wikiToDB.getSentence("selectedSentences", sel[i]);
-            str = wikiToDB.getDBSelectionSentence(sel[i]);  
-            //System.out.println("id=" + sel[i] + str);  
-            selectedLog.println(sel[i] + " " + str);
-            selected_tra_Log.println(sel[i] + " " + str);
-            selected_tra_Log.println(sel[i] + " <" + transcribe(str, "it") + ">");
-          }
-          selectedLog.close();
-          selected_tra_Log.close();
-          
-          logFile.println("Total number of sentences : "+sentIndex);
-        } else
-            System.out.println("No selected sentences to save.");  
-        
-    }
+		// create the selectedSentences table
+		// while the stop criterion is not reached
+		while (!stopCriterionIsReached(selectedIdSents, coverageDefinition)) {
 
-    
-    /*
-     * Utility method for get the Transcription of the selected sentences
-     * It makes use of a started builtin MARY TTS 
-     */
+			// select the next sentence
+			// selectNext(coverageDefinition, logFile, sentIndex, basenameList, vectorArray);
+			boolean haveSelected = selectNext(selectedIdSents, unwantedIdSents, coverageDefinition, cfProvider);
+
+			if (haveSelected) {
+				assert selectedIdSentence >= 0;
+				selectedIdSents.add(selectedIdSentence);
+
+				// print information
+				String msg = "Sentence " + sentIndex + " (" + selectedIdSentence + "), score: " + selectedUsefulness;
+				if (verbose) {
+					System.out.println(msg);
+				}
+				logFile.println(msg);
+				logFile.flush();
+			} else {
+				// nothing more to select
+				// System.out.println("Nothing more to select");
+				logFile.println("Nothing more to select");
+				break;
+			}
+			// the selected sentences will be marked as selected=true in the DB
+			Date date = new Date();
+			System.out.println("  " + sentIndex + " selectedId=" + selectedIdSentence + "  " + fullDate.format(date));
+			// Mark the sentence as selected in dbselection
+			wikiToDB.setSentenceRecord(selectedIdSentence, "selected", true);
+			// Insert selected sentence in table
+			wikiToDB.insertSelectedSentence(selectedIdSentence, false);
+
+			// add the selected sentence to the set
+			// selectedFilenames.add(selectedBasename);
+			// selectedIdSents.add(selectedIdSentence); already done in selectNext
+			// update coverageDefinition
+			coverageDefinition.updateCover(selectedVectors);
+			sentIndex++;
+		}
+		// print out total number of sentences
+		sentIndex--;
+		System.out.println("Total number of selected sentences in TABLE: " + wikiToDB.getSelectedSentencesTableName() + " = "
+				+ sentIndex);
+
+		int sel[] = wikiToDB.getIdListOfType("dbselection", "selected=true and unwanted=false");
+
+		if (sel != null) {
+			// saving sentences in a file
+			System.out.println("Saving selected sentences in ./selected.log");
+			PrintWriter selectedLog = new PrintWriter(new FileWriter(new File("./selected.log")));
+
+			System.out.println("Saving selected sentences and transcriptions in ./selected_text_transcription.log");
+			PrintWriter selected_tra_Log = new PrintWriter(new FileWriter(new File("./selected_text_transcription.log")));
+
+			String str;
+			for (int i = 0; i < sel.length; i++) {
+				// not sure if we need to make another table???
+				// str = wikiToDB.getSentence("selectedSentences", sel[i]);
+				str = wikiToDB.getDBSelectionSentence(sel[i]);
+				// System.out.println("id=" + sel[i] + str);
+				selectedLog.println(sel[i] + " " + str);
+				selected_tra_Log.println(sel[i] + " " + str);
+				selected_tra_Log.println(sel[i] + " <" + transcribe(str, "it") + ">");
+			}
+			selectedLog.close();
+			selected_tra_Log.close();
+
+			logFile.println("Total number of sentences : " + sentIndex);
+		} else
+			System.out.println("No selected sentences to save.");
+
+	}
+
+	/*
+	 * Utility method for get the Transcription of the selected sentences It makes use of a started builtin MARY TTS
+	 */
 	static String transcribe(String ptext, String plocale) throws Exception {
 		String inputType = "TEXT";
 		String outputType = "ALLOPHONES";
@@ -298,48 +292,48 @@ public class SelectionFunction{
 		// System.out.println('<' + lTranscription + '>');
 		return lTranscription;
 	}
-    
 
-    /**
-     * Select the next sentence
-     * 
-     * @param coverageDefinition the coverage definition
-     * @param logFile the logFile 
-     * @param sentenceIndex the index of the next sentence
-     * @param basenameList the list of filenames
-     * @param vectorArray the array of vectors or null
-     *                    if the vectors are on disk
-     * @throws IOException
-     * @return true if a sentence was selected, false otherwise
-     */
-    private boolean selectNext(Set<Integer>selectedIdSents,
-                Set<Integer>unwantedIdSents,
-                CoverageDefinition coverageDefinition,
-                CoverageFeatureProvider cfProvider)
-    throws IOException {
-        // TODO: MS, May 2011 -- I have refactored this code but could not test it. Bad me.
-    	
-        selectedIdSentence = -1;
-        selectedUsefulness = -1;        
+	/**
+	 * Select the next sentence
+	 * 
+	 * @param coverageDefinition
+	 *            the coverage definition
+	 * @param logFile
+	 *            the logFile
+	 * @param sentenceIndex
+	 *            the index of the next sentence
+	 * @param basenameList
+	 *            the list of filenames
+	 * @param vectorArray
+	 *            the array of vectors or null if the vectors are on disk
+	 * @throws IOException
+	 * @return true if a sentence was selected, false otherwise
+	 */
+	private boolean selectNext(Set<Integer> selectedIdSents, Set<Integer> unwantedIdSents, CoverageDefinition coverageDefinition,
+			CoverageFeatureProvider cfProvider) throws IOException {
+		// TODO: MS, May 2011 -- I have refactored this code but could not test it. Bad me.
 
-        // Loop over all sentences in the cfProvider to find the most useful one.
-        // For speed reasons, we need to be a bit smart: if coverage features are not in memory,
-        // we bulk-load a chunk of them at a time.
-        if (cfProvider instanceof InMemoryCFProvider) {
-        	// already in memory, can loop through all
-        	determineMostUsefulSentence(selectedIdSents, unwantedIdSents, coverageDefinition, cfProvider);
-        } else {
-        	assert cfProvider instanceof DatabaseCFProvider;
-        	DatabaseCFProvider dbCfProvider = (DatabaseCFProvider) cfProvider;
-        	int chunkSize = 100000;
-        	for (int c=0, max=dbCfProvider.getNumSentences(); c<max; c+= chunkSize) {
-        		int len = Math.min(chunkSize, max-c);
-        		CoverageFeatureProvider chunk = dbCfProvider.getFeaturesInMemory(c, len);
-        		determineMostUsefulSentence(selectedIdSents, unwantedIdSents, coverageDefinition, chunk);
-        	}
-        }
-        return selectedIdSentence >= 0;
-    }
+		selectedIdSentence = -1;
+		selectedUsefulness = -1;
+
+		// Loop over all sentences in the cfProvider to find the most useful one.
+		// For speed reasons, we need to be a bit smart: if coverage features are not in memory,
+		// we bulk-load a chunk of them at a time.
+		if (cfProvider instanceof InMemoryCFProvider) {
+			// already in memory, can loop through all
+			determineMostUsefulSentence(selectedIdSents, unwantedIdSents, coverageDefinition, cfProvider);
+		} else {
+			assert cfProvider instanceof DatabaseCFProvider;
+			DatabaseCFProvider dbCfProvider = (DatabaseCFProvider) cfProvider;
+			int chunkSize = 100000;
+			for (int c = 0, max = dbCfProvider.getNumSentences(); c < max; c += chunkSize) {
+				int len = Math.min(chunkSize, max - c);
+				CoverageFeatureProvider chunk = dbCfProvider.getFeaturesInMemory(c, len);
+				determineMostUsefulSentence(selectedIdSents, unwantedIdSents, coverageDefinition, chunk);
+			}
+		}
+		return selectedIdSentence >= 0;
+	}
 
 	/**
 	 * @param selectedIdSents
@@ -347,103 +341,93 @@ public class SelectionFunction{
 	 * @param coverageDefinition
 	 * @param cfProvider
 	 */
-	private void determineMostUsefulSentence(Set<Integer> selectedIdSents,
-			Set<Integer> unwantedIdSents,
-			CoverageDefinition coverageDefinition,
-			CoverageFeatureProvider cfProvider) {
-		for (int l=0, num=cfProvider.getNumSentences(); l<num; l++) {
+	private void determineMostUsefulSentence(Set<Integer> selectedIdSents, Set<Integer> unwantedIdSents,
+			CoverageDefinition coverageDefinition, CoverageFeatureProvider cfProvider) {
+		for (int l = 0, num = cfProvider.getNumSentences(); l < num; l++) {
 			int id = cfProvider.getID(l);
 			// skip previously selected or excluded sentences:
 			if (selectedIdSents.contains(id) || unwantedIdSents.contains(id)) {
 				continue;
 			}
 			byte[] nextFeatVects = cfProvider.getCoverageFeatures(l);
-		    //calculate how useful the feature vectors are
-		    double usefulness = coverageDefinition.usefulnessOfFVs(nextFeatVects);
-		   
-		    if (usefulness > selectedUsefulness) {                         
-		        //the current sentence is (currently) the best sentence to add
-		        selectedIdSentence = id;
-		        selectedVectors = nextFeatVects;
-		        selectedUsefulness = usefulness;     
-		    }           
-		    if (usefulness == -1.0){
-		      unwantedIdSents.add(id);
-		      // idSentenceList[i] = -1;     // Here the sentence should be marked as unwanted?
-		      //System.out.println("unwanted id=" + id);
-		    }
+			// calculate how useful the feature vectors are
+			double usefulness = coverageDefinition.usefulnessOfFVs(nextFeatVects);
+
+			if (usefulness > selectedUsefulness) {
+				// the current sentence is (currently) the best sentence to add
+				selectedIdSentence = id;
+				selectedVectors = nextFeatVects;
+				selectedUsefulness = usefulness;
+			}
+			if (usefulness == -1.0) {
+				unwantedIdSents.add(id);
+				// idSentenceList[i] = -1; // Here the sentence should be marked as unwanted?
+				// System.out.println("unwanted id=" + id);
+			}
 		}
 	}
-    
-    
-    /**
-     * Determine if the stop criterion is reached
-     * 
-     * @param sentences the list of selected sentences
-     * @param covDef the coverageDefinition
-     * @return true, if stop criterion is reached, false otherwise
-     */
-    private boolean stopCriterionIsReached(Set<Integer> sentences,
-            CoverageDefinition covDef){
 
-        if (stopNumSentences && sentences.size()>=maxNumSents)
-            //if we have the maximum number of sentences 
-            //stop selecting immediately
-            return true;
-        
-        //other stop criteria can be combined
-        boolean result = false;
-        if (stopSimpleDiphones && covDef.reachedMaxSimpleDiphones())
-            result = true;
-        if (stopSimpleProsody){
-            if (covDef.reachedMaxSimpleProsody()){
-                if (!stopSimpleDiphones && !stopClusteredDiphones){
-                    //set result to true only if we do not have to consider 
-                    //the simpleDiphones or clusteredDiphones stop criterion 
-                    result = true; 
-                }
-                //else result remains false/true, depending on what the 
-                //test result for simpleDiphones/clusteredDiphones was
-            } else {
-                //set the result to false, no matter what the result for
-                //simpleDiphones/clusteredDiphones was
-                result = false;
-            }
-        }
-        return result;
-    }
+	/**
+	 * Determine if the stop criterion is reached
+	 * 
+	 * @param sentences
+	 *            the list of selected sentences
+	 * @param covDef
+	 *            the coverageDefinition
+	 * @return true, if stop criterion is reached, false otherwise
+	 */
+	private boolean stopCriterionIsReached(Set<Integer> sentences, CoverageDefinition covDef) {
 
+		if (stopNumSentences && sentences.size() >= maxNumSents)
+			// if we have the maximum number of sentences
+			// stop selecting immediately
+			return true;
 
+		// other stop criteria can be combined
+		boolean result = false;
+		if (stopSimpleDiphones && covDef.reachedMaxSimpleDiphones())
+			result = true;
+		if (stopSimpleProsody) {
+			if (covDef.reachedMaxSimpleProsody()) {
+				if (!stopSimpleDiphones && !stopClusteredDiphones) {
+					// set result to true only if we do not have to consider
+					// the simpleDiphones or clusteredDiphones stop criterion
+					result = true;
+				}
+				// else result remains false/true, depending on what the
+				// test result for simpleDiphones/clusteredDiphones was
+			} else {
+				// set the result to false, no matter what the result for
+				// simpleDiphones/clusteredDiphones was
+				result = false;
+			}
+		}
+		return result;
+	}
 
-    /**
-     * Read the feature vectors from disk 
-     * for a given filename
-     * 
-     * @param basename the file from which to read from
-     * @return the feature vectors from the file
-     * 
-     * @throws IOException
-     */
-    private byte[] getNextFeatureVectors1(String basename)
-    throws IOException{
-        //open the file
-        FileInputStream fis =
-            new FileInputStream(new File(basename));
-        //read the first 4 bytes and combine them to get the number 
-        //of feature vectors
-        byte[] vlength = new byte[4];
-        fis.read(vlength);
-        int numFeatVects = (((vlength[0] & 0xff) << 24) 
-                | ((vlength[1] & 0xff) << 16) 
-                | ((vlength[2] & 0xff) << 8)
-                | (vlength[3] & 0xff));
-        //read the content of the file into a byte array
-        byte[] vectorBuf = new byte[4*numFeatVects];
-        fis.read(vectorBuf);
-        fis.close();        
-        //return the feature vectors
-        return vectorBuf;
-    }
-
+	/**
+	 * Read the feature vectors from disk for a given filename
+	 * 
+	 * @param basename
+	 *            the file from which to read from
+	 * @return the feature vectors from the file
+	 * 
+	 * @throws IOException
+	 */
+	private byte[] getNextFeatureVectors1(String basename) throws IOException {
+		// open the file
+		FileInputStream fis = new FileInputStream(new File(basename));
+		// read the first 4 bytes and combine them to get the number
+		// of feature vectors
+		byte[] vlength = new byte[4];
+		fis.read(vlength);
+		int numFeatVects = (((vlength[0] & 0xff) << 24) | ((vlength[1] & 0xff) << 16) | ((vlength[2] & 0xff) << 8) | (vlength[3] & 0xff));
+		// read the content of the file into a byte array
+		byte[] vectorBuf = new byte[4 * numFeatVects];
+		fis.read(vectorBuf);
+		fis.close();
+		// return the feature vectors
+		return vectorBuf;
+	}
 
 }

marytts-builder/src/main/java/marytts/tools/dbselection/WikipediaProcessor.java

@@ -28,357 +28,433 @@ import java.io.RandomAccessFile;
 import java.util.Scanner;
 import java.util.Vector;
 
-
 /**
- * WikipediaProcessor
- * This program processes one by one the xml files split with wikipediaDumpSplitter.
- * Each xml file is converted to an sql source file with mwdumper-2008-04-13.jar (org.mediawiki.dumper.Dumper)
- * The tables names in the sql source are prefixed with the local (ex. en_US, de etc.) 
- * Each sql source is loaded in a mysql database, basically the tables local_text, local_page and local_revision are loaded.
- * Once the tables are loaded the WikipediMarkupCleaner is used to extract clean text and a wordList, as a 
- * result two tables will be created in the database: local_cleanText and local_wordList (the wordList is also 
- * saved in a file).
+ * WikipediaProcessor This program processes one by one the xml files split with wikipediaDumpSplitter. Each xml file is converted
+ * to an sql source file with mwdumper-2008-04-13.jar (org.mediawiki.dumper.Dumper) The tables names in the sql source are
+ * prefixed with the local (ex. en_US, de etc.) Each sql source is loaded in a mysql database, basically the tables local_text,
+ * local_page and local_revision are loaded. Once the tables are loaded the WikipediMarkupCleaner is used to extract clean text
+ * and a wordList, as a result two tables will be created in the database: local_cleanText and local_wordList (the wordList is
+ * also saved in a file).
  * 
  * @author Marcela Charfuelan.
  */
 public class WikipediaProcessor {
-    
-    // locale
-    private String locale=null;
-    // mySql database 
-    private String mysqlHost=null;
-    private String mysqlDB=null;
-    private String mysqlUser=null;
-    private String mysqlPasswd=null;
-    // Wikipedia files:
-    private String listFile=null;
-    private String textFile=null;
-    private String pageFile=null;
-    private String revisionFile=null;
-    private String wikiLog = null;
-    private boolean debug = false;
-    private String debugPageId = null;
-    // Default settings for max page length and min and max text length
-    private int minPageLength=10000;  // minimum size of a wikipedia page, to be used in the first filtering of pages
-    private int minTextLength=1000;
-    private int maxTextLength=15000;  // the average lenght in one big xml file is approx. 12000
-    
-    // Use this variable to save time not loading Wiki tables, if they already exist in the DB
-    private boolean loadWikiTables = true;
-   
-    // Use this variable to do not create a new cleanText table, but adding to an already existing cleanText table.
-    private boolean deleteCleanTextTable = false;
-    
-    public void setLocale(String str){ locale = str; }
-    public void setMysqlHost(String str){ mysqlHost = str; }
-    public void setMysqlDB(String str){ mysqlDB = str; }
-    public void setMysqlUser(String str){ mysqlUser = str; }
-    public void setMysqlPasswd(String str){ mysqlPasswd = str; }
-   
-    public void setListFile(String str){ listFile = str; }
-    public void setTextFile(String str){ textFile = str; }
-    public void setPageFile(String str){ pageFile = str; }
-    public void setRevisionFile(String str){ revisionFile = str; }
-    public void setWikiLog(String str){ wikiLog = str; }
-    public void setTestId(String str){ debugPageId = str; }
-    
-    public void setMinPageLength(int val){ minPageLength = val; }
-    public void setMinTextLength(int val){ minTextLength = val; }
-    public void setMaxTextLength(int val){ maxTextLength = val; }
-    
-    public void setDebug(boolean bval){ debug = bval; }
-    public void setLoadWikiTables(boolean bval){ loadWikiTables = bval; }
-    public void setDeleteCleanTextTable(boolean bval){ deleteCleanTextTable = bval; }
-       
-    public String getLocale(){ return locale; }
-    public String getMysqlHost(){ return mysqlHost; }
-    public String getMysqlDB(){ return mysqlDB; }
-    public String getMysqlUser(){ return mysqlUser; }
-    public String getMysqlPasswd(){ return mysqlPasswd; }
-    
-    public String getListFile(){ return listFile; }
-    public String getTextFile(){ return textFile; }
-    public String getPageFile(){ return pageFile; }
-    public String getRevisionFile(){ return revisionFile; }
-    public String getWikiLog(){ return wikiLog; }
-    public String getTestId(){ return debugPageId; }
-    
-    public int getMinPageLength(){ return minPageLength; }
-    public int getMinTextLength(){ return minTextLength; }
-    public int getMaxTextLength(){ return maxTextLength; }
-    
-    public boolean getDebug(){ return debug; }
-    public boolean getLoadWikiTables(){ return loadWikiTables; }
-    public boolean getDeleteCleanTextTable(){ return deleteCleanTextTable; }
-    
 
-    private void printParameters(){
-        System.out.println("WikipediaMarkupCleaner parameters:" +
-        "\n  -mysqlHost " + getMysqlHost() +
-        "\n  -mysqlUser " + getMysqlUser() +
-        "\n  -mysqlPasswd " + getMysqlPasswd() +
-        "\n  -mysqlDB " + getMysqlDB() +
-        "\n  -listFile " + getListFile() +
-        "\n  -minPage " + getMinPageLength() +
-        "\n  -minText " + getMinTextLength() +
-        "\n  -maxText " + getMaxTextLength() );
-        
-    }
-    
-    
-    /**
-     * Read and parse the command line args
-     * 
-     * @param args the args
-     * @return true, if successful, false otherwise
-     */
-    private boolean readArgs(String[] args){
-        
-        String help = "\nUsage: java WikipediaProcessor -locale language -mysqlHost host -mysqlUser user -mysqlPasswd passwd \n" +
-        "                                   -mysqlDB wikiDB -listFile wikiFileList.\n" +
-        "                                   [-minPage 10000 -minText 1000 -maxText 15000] \n\n" +
-        "      -listFile is a a text file that contains the xml wikipedia file names to be procesed. \n" +
-        "      This program requires the jar file mwdumper-2008-04-13.jar (or latest). \n\n" +
-        "      default/optional: [-minPage 10000 -minText 1000 -maxText 15000] \n" +
-        "      -minPage is the minimum size of a wikipedia page that will be considered for cleaning.\n" +
-        "      -minText is the minimum size of a text to be kept in the DB.\n" +
-        "      -maxText is used to split big articles in small chunks, this is the maximum chunk size. \n";
-              
-        if (args.length >= 12){  // minimum 12 parameters
-          for(int i=0; i<args.length; i++) { 
-            if(args[i].contentEquals("-locale") && args.length >= (i+1) )
-              setLocale(args[++i]);
-              
-            else if(args[i].contentEquals("-mysqlHost") && args.length >= (i+1) )
-              setMysqlHost(args[++i]);
-            
-            else if(args[i].contentEquals("-mysqlUser") && args.length >= (i+1) )
-               setMysqlUser(args[++i]);
-              
-            else if(args[i].contentEquals("-mysqlPasswd") && args.length >= (i+1) )
-               setMysqlPasswd(args[++i]);
-            
-            else if(args[i].contentEquals("-mysqlDB") && args.length >= (i+1) )
-              setMysqlDB(args[++i]);
-                      
-            else if(args[i].contentEquals("-listFile") && args.length >= (i+1) )
-                setListFile(args[++i]);
-            
-            // From here the arguments are optional
-            else if(args[i].contentEquals("-minPage") && args.length >= (i+1) )
-              setMinPageLength(Integer.parseInt(args[++i]));
-            
-            else if(args[i].contentEquals("-minText") && args.length >= (i+1) )
-              setMinTextLength(Integer.parseInt(args[++i]));
-            
-            else if(args[i].contentEquals("-maxText") && args.length >= (i+1) )
-             setMaxTextLength(Integer.parseInt(args[++i]));
-            
-            else { //unknown argument
-                System.out.println("\nOption not known: " + args[i]);
-                System.out.println(help);
-                return false;
-              }
-            
-          }
-       } else { // num arguments less than 12
-          System.out.println(help);
-          return false;
-        }
-        
-        if(getMysqlHost()==null || getMysqlUser()==null || getMysqlPasswd()==null || getMysqlDB()==null){
-            System.out.println("\nMissing required mysql parameters (one/several required variables are null).");
-            printParameters();
-            System.out.println(help);
-            return false;
-         } 
-        
-        if(getListFile()==null){
-            System.out.println("\nMissing required parameter -listFile wikiFileList.\n");
-            printParameters();
-            System.out.println(help);
-            return false; 
-        }
-        
-        
-      return true;  
-    }
-    
-    private Vector<String> getWikipediaFiles(String fileName)throws Exception {
-        
-      BufferedReader in = null;
-      String line;
-      Vector<String> files = null;
-      
-      // check if the file exist
-      File f  = new File(fileName);
-      if(f.exists())
-      {  
-        files = new Vector<String>();    
-        try {
-            in = new BufferedReader(new FileReader(fileName)); 
-            while ((line = in.readLine()) != null) {
-              files.add(line);
-            }
-            in.close();
-          
-        } catch( Exception e ) {
-          e.printStackTrace();
-        }       
-      } 
-      
-      return files;
-      
-    }
-    
+	// locale
+	private String locale = null;
+	// mySql database
+	private String mysqlHost = null;
+	private String mysqlDB = null;
+	private String mysqlUser = null;
+	private String mysqlPasswd = null;
+	// Wikipedia files:
+	private String listFile = null;
+	private String textFile = null;
+	private String pageFile = null;
+	private String revisionFile = null;
+	private String wikiLog = null;
+	private boolean debug = false;
+	private String debugPageId = null;
+	// Default settings for max page length and min and max text length
+	private int minPageLength = 10000; // minimum size of a wikipedia page, to be used in the first filtering of pages
+	private int minTextLength = 1000;
+	private int maxTextLength = 15000; // the average lenght in one big xml file is approx. 12000
+
+	// Use this variable to save time not loading Wiki tables, if they already exist in the DB
+	private boolean loadWikiTables = true;
+
+	// Use this variable to do not create a new cleanText table, but adding to an already existing cleanText table.
+	private boolean deleteCleanTextTable = false;
+
+	public void setLocale(String str) {
+		locale = str;
+	}
+
+	public void setMysqlHost(String str) {
+		mysqlHost = str;
+	}
+
+	public void setMysqlDB(String str) {
+		mysqlDB = str;
+	}
+
+	public void setMysqlUser(String str) {
+		mysqlUser = str;
+	}
+
+	public void setMysqlPasswd(String str) {
+		mysqlPasswd = str;
+	}
+
+	public void setListFile(String str) {
+		listFile = str;
+	}
+
+	public void setTextFile(String str) {
+		textFile = str;
+	}
+
+	public void setPageFile(String str) {
+		pageFile = str;
+	}
+
+	public void setRevisionFile(String str) {
+		revisionFile = str;
+	}
+
+	public void setWikiLog(String str) {
+		wikiLog = str;
+	}
+
+	public void setTestId(String str) {
+		debugPageId = str;
+	}
+
+	public void setMinPageLength(int val) {
+		minPageLength = val;
+	}
+
+	public void setMinTextLength(int val) {
+		minTextLength = val;
+	}
+
+	public void setMaxTextLength(int val) {
+		maxTextLength = val;
+	}
+
+	public void setDebug(boolean bval) {
+		debug = bval;
+	}
+
+	public void setLoadWikiTables(boolean bval) {
+		loadWikiTables = bval;
+	}
+
+	public void setDeleteCleanTextTable(boolean bval) {
+		deleteCleanTextTable = bval;
+	}
+
+	public String getLocale() {
+		return locale;
+	}
+
+	public String getMysqlHost() {
+		return mysqlHost;
+	}
+
+	public String getMysqlDB() {
+		return mysqlDB;
+	}
+
+	public String getMysqlUser() {
+		return mysqlUser;
+	}
+
+	public String getMysqlPasswd() {
+		return mysqlPasswd;
+	}
+
+	public String getListFile() {
+		return listFile;
+	}
+
+	public String getTextFile() {
+		return textFile;
+	}
+
+	public String getPageFile() {
+		return pageFile;
+	}
+
+	public String getRevisionFile() {
+		return revisionFile;
+	}
+
+	public String getWikiLog() {
+		return wikiLog;
+	}
+
+	public String getTestId() {
+		return debugPageId;
+	}
+
+	public int getMinPageLength() {
+		return minPageLength;
+	}
+
+	public int getMinTextLength() {
+		return minTextLength;
+	}
+
+	public int getMaxTextLength() {
+		return maxTextLength;
+	}
+
+	public boolean getDebug() {
+		return debug;
+	}
+
+	public boolean getLoadWikiTables() {
+		return loadWikiTables;
+	}
+
+	public boolean getDeleteCleanTextTable() {
+		return deleteCleanTextTable;
+	}
+
+	private void printParameters() {
+		System.out.println("WikipediaMarkupCleaner parameters:" + "\n  -mysqlHost " + getMysqlHost() + "\n  -mysqlUser "
+				+ getMysqlUser() + "\n  -mysqlPasswd " + getMysqlPasswd() + "\n  -mysqlDB " + getMysqlDB() + "\n  -listFile "
+				+ getListFile() + "\n  -minPage " + getMinPageLength() + "\n  -minText " + getMinTextLength() + "\n  -maxText "
+				+ getMaxTextLength());
+
+	}
+
+	/**
+	 * Read and parse the command line args
+	 * 
+	 * @param args
+	 *            the args
+	 * @return true, if successful, false otherwise
+	 */
+	private boolean readArgs(String[] args) {
+
+		String help = "\nUsage: java WikipediaProcessor -locale language -mysqlHost host -mysqlUser user -mysqlPasswd passwd \n"
+				+ "                                   -mysqlDB wikiDB -listFile wikiFileList.\n"
+				+ "                                   [-minPage 10000 -minText 1000 -maxText 15000] \n\n"
+				+ "      -listFile is a a text file that contains the xml wikipedia file names to be procesed. \n"
+				+ "      This program requires the jar file mwdumper-2008-04-13.jar (or latest). \n\n"
+				+ "      default/optional: [-minPage 10000 -minText 1000 -maxText 15000] \n"
+				+ "      -minPage is the minimum size of a wikipedia page that will be considered for cleaning.\n"
+				+ "      -minText is the minimum size of a text to be kept in the DB.\n"
+				+ "      -maxText is used to split big articles in small chunks, this is the maximum chunk size. \n";
+
+		if (args.length >= 12) { // minimum 12 parameters
+			for (int i = 0; i < args.length; i++) {
+				if (args[i].contentEquals("-locale") && args.length >= (i + 1))
+					setLocale(args[++i]);
+
+				else if (args[i].contentEquals("-mysqlHost") && args.length >= (i + 1))
+					setMysqlHost(args[++i]);
+
+				else if (args[i].contentEquals("-mysqlUser") && args.length >= (i + 1))
+					setMysqlUser(args[++i]);
+
+				else if (args[i].contentEquals("-mysqlPasswd") && args.length >= (i + 1))
+					setMysqlPasswd(args[++i]);
+
+				else if (args[i].contentEquals("-mysqlDB") && args.length >= (i + 1))
+					setMysqlDB(args[++i]);
+
+				else if (args[i].contentEquals("-listFile") && args.length >= (i + 1))
+					setListFile(args[++i]);
+
+				// From here the arguments are optional
+				else if (args[i].contentEquals("-minPage") && args.length >= (i + 1))
+					setMinPageLength(Integer.parseInt(args[++i]));
+
+				else if (args[i].contentEquals("-minText") && args.length >= (i + 1))
+					setMinTextLength(Integer.parseInt(args[++i]));
+
+				else if (args[i].contentEquals("-maxText") && args.length >= (i + 1))
+					setMaxTextLength(Integer.parseInt(args[++i]));
+
+				else { // unknown argument
+					System.out.println("\nOption not known: " + args[i]);
+					System.out.println(help);
+					return false;
+				}
+
+			}
+		} else { // num arguments less than 12
+			System.out.println(help);
+			return false;
+		}
+
+		if (getMysqlHost() == null || getMysqlUser() == null || getMysqlPasswd() == null || getMysqlDB() == null) {
+			System.out.println("\nMissing required mysql parameters (one/several required variables are null).");
+			printParameters();
+			System.out.println(help);
+			return false;
+		}
+
+		if (getListFile() == null) {
+			System.out.println("\nMissing required parameter -listFile wikiFileList.\n");
+			printParameters();
+			System.out.println(help);
+			return false;
+		}
+
+		return true;
+	}
+
+	private Vector<String> getWikipediaFiles(String fileName) throws Exception {
+
+		BufferedReader in = null;
+		String line;
+		Vector<String> files = null;
+
+		// check if the file exist
+		File f = new File(fileName);
+		if (f.exists()) {
+			files = new Vector<String>();
+			try {
+				in = new BufferedReader(new FileReader(fileName));
+				while ((line = in.readLine()) != null) {
+					files.add(line);
+				}
+				in.close();
+
+			} catch (Exception e) {
+				e.printStackTrace();
+			}
+		}
+
+		return files;
+
+	}
+
+	private void setWikipediaFileDone(String fileName, String fileDone) {
+
+		RandomAccessFile out = null;
+
+		try {
+			out = new RandomAccessFile(fileName, "rw");
+			out.seek(out.length());
+			out.writeBytes(fileDone + "\n");
+			out.close();
+
+		} catch (Exception e) {
+			e.printStackTrace();
+		}
+
+	}
+
+	private void addLocalePrefixToTables(String sqlFile, String outFile) {
+		String line, localLine;
+		Scanner s = null;
+		FileWriter outputStream = null;
+
+		try {
+			s = new Scanner(new BufferedReader(new FileReader(sqlFile)));
+
+			System.out.println("Adding local prefix to sql tables.");
+			outputStream = new FileWriter(outFile);
+
+			while (s.hasNext()) {
+				line = s.nextLine();
+
+				if (line.contains("INSERT INTO ")) {
+					localLine = line.replaceAll("INSERT INTO ", "INSERT INTO " + locale + "_");
+					outputStream.write(localLine + "\n");
+				} else
+					outputStream.write(line + "\n");
+
+			}
+			outputStream.close();
+			System.out.println("Added local=" + locale + " to tables in outFile:" + outFile);
+
+		} catch (Exception e) {
+			System.err.println("Exception: " + e.getMessage());
+		} finally {
+			if (s != null)
+				s.close();
+		}
+
+	}
+
+	public static void main(String[] args) throws Exception {
+		String wFile; // xml wiki file
+		String doneFile = "./done.txt"; // file that contains the xml files already processed
+		Vector<String> filesToProcess;
+		Vector<String> filesDone;
+		WikipediaProcessor wiki = new WikipediaProcessor();
+
+		/* check the arguments */
+		if (!wiki.readArgs(args))
+			return;
+		wiki.printParameters();
+
+		// checking if cleanText table exist
+		DBHandler wikiToDB = new DBHandler(wiki.getLocale());
+		wikiToDB.createDBConnection(wiki.getMysqlHost(), wiki.getMysqlDB(), wiki.getMysqlUser(), wiki.getMysqlPasswd());
+		char c;
+		boolean result = false, processFiles = true;
+		InputStreamReader isr = new InputStreamReader(System.in);
+		BufferedReader br = new BufferedReader(isr);
+
+		String table = wiki.getLocale() + "_cleanText";
+		if (wikiToDB.tableExist(table)) {
+			System.out.print("    TABLE = \"" + table + "\" already exists, should it be deleted (y/n)?");
+			try {
+				String s = br.readLine();
+				if (s.contentEquals("y")) {
+					wikiToDB.createWikipediaCleanTextTable();
+				} else {
+					System.out.print("    ADDING clean text TO EXISTING cleanText TABLE \"" + wiki.getLocale()
+							+ "_cleanText\" (y/n)?");
+					s = br.readLine();
+					if (s.contentEquals("y"))
+						processFiles = true;
+					else {
+						processFiles = false;
+						System.out
+								.print("    please check the \"locale\" prefix of the locale_cleanText TABLE you want to create or add to.");
+					}
+				}
+			} catch (Exception e) {
+				System.out.println(e);
+			}
+		} else
+			System.out.print("    TABLE = \"" + table + "\" does not exist, it will be created.");
+		wikiToDB.closeDBConnection();
+
+		if (processFiles) {
+			filesToProcess = wiki.getWikipediaFiles(wiki.getListFile());
+			filesDone = wiki.getWikipediaFiles(doneFile);
+			if (filesDone == null)
+				filesDone = new Vector<String>();
+
+			if (filesToProcess != null) {
+				for (int i = 0; i < filesToProcess.size(); i++) {
+					wFile = filesToProcess.elementAt(i);
+					if (filesDone.indexOf(wFile) == -1) {
+						System.out.println("\n_______________________________________________________________________________");
+
+						System.out.println("\nProcessing xml file:" + wFile);
+
+						WikipediaMarkupCleaner wikiCleaner = new WikipediaMarkupCleaner();
+
+						// Set parameters in the WikipediaMarkupCleaner
+						wikiCleaner.setDebug(false);
+						wikiCleaner.setDeleteCleanTextTable(false);
+						wikiCleaner.setLoadWikiTables(true);
+						wikiCleaner.setLocale(wiki.getLocale());
+
+						wikiCleaner.setMaxTextLength(wiki.getMaxTextLength());
+						wikiCleaner.setMinPageLength(wiki.getMinPageLength());
+						wikiCleaner.setMinTextLength(wiki.getMinTextLength());
+						wikiCleaner.setMysqlDB(wiki.getMysqlDB());
+						wikiCleaner.setMysqlHost(wiki.getMysqlHost());
+						wikiCleaner.setMysqlPasswd(wiki.getMysqlPasswd());
+						wikiCleaner.setMysqlUser(wiki.getMysqlUser());
+
+						// process xml file
+						wikiCleaner.setXmlWikiFile(wFile);
+						wikiCleaner.processWikipediaPages();
+						wikiCleaner = null;
+
+						// when finished
+						wiki.setWikipediaFileDone("./done.txt", wFile);
+
+					} else
+						System.out.println("File already procesed: " + wFile);
+				}
+			} else
+				System.out.println("Empty list of files to process.");
+		} else
+			System.out.println("WikipediaProcessor terminated.");
+
+	}
 
-    private void setWikipediaFileDone(String fileName, String fileDone){
-        
-        RandomAccessFile out = null;
-        
-        try {
-            out = new RandomAccessFile(fileName, "rw");
-            out.seek(out.length());
-            out.writeBytes(fileDone+"\n");
-            out.close();
-            
-        } catch( Exception e ) {
-            e.printStackTrace();
-        }       
-        
-      }
-    
-    
-    private void addLocalePrefixToTables(String sqlFile, String outFile){
-        String line, localLine;
-        Scanner s = null;
-        FileWriter outputStream = null;
-        
-        try {
-          s = new Scanner(new BufferedReader(new FileReader(sqlFile)));
-          
-          
-          System.out.println("Adding local prefix to sql tables.");
-          outputStream = new FileWriter(outFile);
-          
-          
-          while (s.hasNext()) {
-            line = s.nextLine();
-            
-            if(line.contains("INSERT INTO ")){
-               localLine = line.replaceAll("INSERT INTO ", "INSERT INTO " + locale + "_");
-               outputStream.write(localLine+"\n");
-            } else
-                outputStream.write(line+"\n");  
-            
-          }
-          outputStream.close();
-          System.out.println("Added local=" + locale + " to tables in outFile:" + outFile);
-          
-        }catch (Exception e) {
-            System.err.println("Exception: " + e.getMessage());
-        } finally {
-            if (s != null)
-              s.close();
-        }   
-        
-        
-    }
-    
-    
-    public static void main(String[] args) throws Exception{
-        String wFile;                    // xml wiki file
-        String doneFile = "./done.txt";  // file that contains the xml files already processed
-        Vector<String> filesToProcess;
-        Vector<String> filesDone;
-        WikipediaProcessor wiki = new WikipediaProcessor(); 
-        
-        /* check the arguments */
-        if (!wiki.readArgs(args))
-            return;
-        wiki.printParameters();
-        
-        // checking if cleanText table exist
-        DBHandler wikiToDB = new DBHandler(wiki.getLocale());
-        wikiToDB.createDBConnection(wiki.getMysqlHost(),wiki.getMysqlDB(),wiki.getMysqlUser(),wiki.getMysqlPasswd());
-        char c;
-        boolean result=false, processFiles=true;
-        InputStreamReader isr = new InputStreamReader(System.in);
-        BufferedReader br = new BufferedReader(isr);
-    
-        String table = wiki.getLocale() + "_cleanText";
-        if(wikiToDB.tableExist(table)) {
-          System.out.print("    TABLE = \"" + table + "\" already exists, should it be deleted (y/n)?"); 
-          try{
-            String s = br.readLine();  
-            if( s.contentEquals("y")){
-              wikiToDB.createWikipediaCleanTextTable();
-            } else {                
-              System.out.print("    ADDING clean text TO EXISTING cleanText TABLE \"" + wiki.getLocale() + "_cleanText\" (y/n)?");
-              s = br.readLine();
-              if( s.contentEquals("y"))
-                processFiles=true;
-              else{
-                processFiles=false;
-                System.out.print("    please check the \"locale\" prefix of the locale_cleanText TABLE you want to create or add to.");
-              }
-            }        
-        } catch(Exception e){
-            System.out.println(e); 
-        }
-        } else 
-          System.out.print("    TABLE = \"" + table + "\" does not exist, it will be created."); 
-        wikiToDB.closeDBConnection();
-        
-        if(processFiles){
-        filesToProcess = wiki.getWikipediaFiles(wiki.getListFile());
-        filesDone = wiki.getWikipediaFiles(doneFile);
-        if(filesDone==null)
-           filesDone = new Vector<String>();
-        
-        if(filesToProcess != null){
-          for(int i=0; i<filesToProcess.size(); i++){
-            wFile = filesToProcess.elementAt(i);
-            if(filesDone.indexOf(wFile) == -1){
-               System.out.println("\n_______________________________________________________________________________"); 
-               
-               System.out.println("\nProcessing xml file:" + wFile);
-              
-               WikipediaMarkupCleaner wikiCleaner = new WikipediaMarkupCleaner();
-               
-               // Set parameters in the WikipediaMarkupCleaner
-               wikiCleaner.setDebug(false);
-               wikiCleaner.setDeleteCleanTextTable(false);
-               wikiCleaner.setLoadWikiTables(true);
-               wikiCleaner.setLocale(wiki.getLocale());
-               
-               wikiCleaner.setMaxTextLength(wiki.getMaxTextLength());
-               wikiCleaner.setMinPageLength(wiki.getMinPageLength());
-               wikiCleaner.setMinTextLength(wiki.getMinTextLength());
-               wikiCleaner.setMysqlDB(wiki.getMysqlDB());
-               wikiCleaner.setMysqlHost(wiki.getMysqlHost());
-               wikiCleaner.setMysqlPasswd(wiki.getMysqlPasswd());
-               wikiCleaner.setMysqlUser(wiki.getMysqlUser());
-           
-               // process xml file
-               wikiCleaner.setXmlWikiFile(wFile);
-               wikiCleaner.processWikipediaPages();              
-               wikiCleaner = null;
-               
-               // when finished
-               wiki.setWikipediaFileDone("./done.txt", wFile);
-      
-            } else
-              System.out.println("File already procesed: " + wFile);           
-          }
-        } else
-          System.out.println("Empty list of files to process.");
-        } else 
-          System.out.println("WikipediaProcessor terminated.");   
-            
-    }
-    
-    
-    
 }
-

marytts-builder/src/main/java/marytts/tools/newlanguage/LTSTrainer.java

@@ -57,11 +57,9 @@ import weka.core.Instances;
 
 /**
  * 
- * This class is a generic approach to predict a phone sequence from a 
- * grapheme sequence. 
+ * This class is a generic approach to predict a phone sequence from a grapheme sequence.
  * 
- * the normal sequence of steps is:
- * 1) initialize the trainer with a phone set and a locale
+ * the normal sequence of steps is: 1) initialize the trainer with a phone set and a locale
  * 
  * 2) read in the lexicon, preserve stress if you like
  * 
@@ -77,367 +75,368 @@ import weka.core.Instances;
  *
  */
 
-public class LTSTrainer extends AlignerTrainer
-{
+public class LTSTrainer extends AlignerTrainer {
 
+	protected AllophoneSet phSet;
 
-    protected AllophoneSet phSet;
-    
-    protected int context;
-    protected boolean convertToLowercase;
-    protected boolean considerStress;
-    
-    /**
-     * Create a new LTSTrainer.
-     * @param aPhSet the allophone set to use.
-     * @param convertToLowercase whether to convert all graphemes to lowercase, using the locale of the allophone set.
-     * @param considerStress indicator if stress is preserved
-     */
-    public LTSTrainer(AllophoneSet aPhSet, boolean convertToLowercase, boolean considerStress, int context) {
-        super();
-        this.phSet = aPhSet;
-        this.convertToLowercase = convertToLowercase;
-        this.considerStress = considerStress;
-        this.context = context;
-        BasicConfigurator.configure();
-    }
-    
-    /**
-     * Train the tree, using binary decision nodes. 
-     * @param minLeafData the minimum number of instances that have to occur in at least two subsets induced by split
-     * @return
-     * @throws IOException
-     */
-    public CART trainTree(int minLeafData) throws IOException{
-        
-        Map<String, List<String[]>> grapheme2align = new HashMap<String, List<String[]>>();
-        for (String gr : this.graphemeSet){      
-            grapheme2align.put(gr, new ArrayList<String[]>());
-        }
-        
-        Set<String> phChains = new HashSet<String>();
-        
-        // for every alignment pair collect counts
-        for ( int i = 0; i < this.inSplit.size(); i++ ){
-            
-            StringPair[] alignment = this.getAlignment(i);
-            
-            for ( int inNr = 0 ; inNr < alignment.length ; inNr ++ ){
-                
-                //System.err.println(alignment[inNr]);
-                
-                // quotation signs needed to represent empty string
-                String outAlNr = "'" + alignment[inNr].getString2() + "'";
-                
-                // TODO: don't consider alignments to more than three characters
-                if (outAlNr.length() > 5)
-                    continue;
+	protected int context;
+	protected boolean convertToLowercase;
+	protected boolean considerStress;
 
-                phChains.add(outAlNr);
-                
-                // storing context and target
-                String[] datapoint = new String[2*context + 2];
-                
-                for (int ct = 0; ct < 2*context+1; ct++){
-                    int pos = inNr - context +ct;
-                    
-                    if (pos >=0  && pos < alignment.length){
-                        datapoint[ct] = alignment[pos].getString1();
-                    } else {
-                        datapoint[ct] = "null";
-                    }
-                    
-                }
-                
-                // set target
-                datapoint[2*context+1] = outAlNr;
-                
-                // add datapoint
-                grapheme2align.get(alignment[inNr].getString1()).add(datapoint);                                
-            }
-        }
-        
-        // for conversion need feature definition file        
-        FeatureDefinition fd = this.graphemeFeatureDef(phChains);
-        
-        int centerGrapheme = fd.getFeatureIndex("att"+(context+1));
-        
-        List<CART> stl = new ArrayList<CART>(fd.getNumberOfValues(centerGrapheme));
-        
-        for (String gr : fd.getPossibleValues(centerGrapheme)){
-            System.out.println("      Training decision tree for: " + gr);
-            logger.debug("      Training decision tree for: " + gr);
-            
-            ArrayList<Attribute> attributeDeclarations = new ArrayList<Attribute>();
-            
-            // attributes with values
-            for (int att = 1; att <= context*2 + 1; att++){
-                
-                // ...collect possible values
-                ArrayList<String> attVals = new ArrayList<String>();
-                                
-                String featureName = "att"+att;
-                
-                for (String usableGrapheme:fd.getPossibleValues(fd.getFeatureIndex(featureName))){
-                    attVals.add(usableGrapheme);
-                }
-                
-                attributeDeclarations.add(new Attribute(featureName, attVals) );
-            }
-            
-            List<String[]> datapoints = grapheme2align.get(gr);
-            
-            // maybe training is faster with targets limited to grapheme
-            Set<String> graphSpecPh = new HashSet<String>();
-            for (String[] dp : datapoints){
-                graphSpecPh.add(dp[dp.length-1]);
-            }
-            
-            // targetattribute
-            // ...collect possible values
-            ArrayList<String> targetVals = new ArrayList<String>();
-            for (String phc : graphSpecPh){// todo: use either fd of phChains
-                targetVals.add(phc);
-            }
-            attributeDeclarations.add(new Attribute(TrainedLTS.PREDICTED_STRING_FEATURENAME, targetVals) );
+	/**
+	 * Create a new LTSTrainer.
+	 * 
+	 * @param aPhSet
+	 *            the allophone set to use.
+	 * @param convertToLowercase
+	 *            whether to convert all graphemes to lowercase, using the locale of the allophone set.
+	 * @param considerStress
+	 *            indicator if stress is preserved
+	 */
+	public LTSTrainer(AllophoneSet aPhSet, boolean convertToLowercase, boolean considerStress, int context) {
+		super();
+		this.phSet = aPhSet;
+		this.convertToLowercase = convertToLowercase;
+		this.considerStress = considerStress;
+		this.context = context;
+		BasicConfigurator.configure();
+	}
 
-            // now, create the dataset adding the datapoints
-            Instances data = new Instances(gr, attributeDeclarations, 0);
-            
-            // datapoints
-            for (String[] point : datapoints){
-                
-                Instance currInst = new DenseInstance( data.numAttributes()  );
-                currInst.setDataset(data);
-                
-                for (int i = 0; i < point.length; i++){
-                                       
-                    currInst.setValue(i, point[i]);
-                }
-                
-                data.add(currInst);
-            }
-            
-            // Make the last attribute be the class
-            data.setClassIndex(data.numAttributes() - 1);
-            
-            // build the tree without using the J48 wrapper class
-            // standard parameters are: 
-            // binary split selection with minimum x instances at the leaves, tree is pruned, confidenced value, subtree raising, cleanup, don't collapse
-            // Here is used a modifed version of C45PruneableClassifierTree that allow using Unary Classes (see Issue #51)
-            C45PruneableClassifierTree decisionTree;
-            try {
-                decisionTree = new C45PruneableClassifierTreeWithUnary(new BinC45ModelSelection(minLeafData,data,true),true,0.25f,true,true, false);
-                decisionTree.buildClassifier(data);
-            } catch (Exception e) {
-                throw new RuntimeException("couldn't train decisiontree using weka: ", e);
-            }
-            
-            CART maryTree = TreeConverter.c45toStringCART(decisionTree, fd,data);
-            
-            stl.add(maryTree);
-       }
+	/**
+	 * Train the tree, using binary decision nodes.
+	 * 
+	 * @param minLeafData
+	 *            the minimum number of instances that have to occur in at least two subsets induced by split
+	 * @return
+	 * @throws IOException
+	 */
+	public CART trainTree(int minLeafData) throws IOException {
 
-        DecisionNode.ByteDecisionNode rootNode = new DecisionNode.ByteDecisionNode(centerGrapheme, stl.size(), fd);
-        for (CART st : stl) {
-            rootNode.addDaughter(st.getRootNode());
-        }
+		Map<String, List<String[]>> grapheme2align = new HashMap<String, List<String[]>>();
+		for (String gr : this.graphemeSet) {
+			grapheme2align.put(gr, new ArrayList<String[]>());
+		}
 
-        Properties props = new Properties();
-        props.setProperty("lowercase", String.valueOf(convertToLowercase));
-        props.setProperty("stress", String.valueOf(considerStress));
-        props.setProperty("context", String.valueOf(context));
-        
-        CART bigTree = new CART(rootNode, fd, props);
-                        
-        return bigTree;
-    }
-    
-    /**
-     * 
-     * Convenience method to save files to graph2phon.wagon and graph2phon.pfeats
-     * in a specified directory with UTF-8 encoding.
-     * 
-     * @param tree
-     * @param saveTreePath
-     * @throws IOException
-     */
-    public void save(CART tree, String saveTreefile) throws IOException{
-        MaryCARTWriter mcw = new MaryCARTWriter();
-        mcw.dumpMaryCART(tree, saveTreefile);
-    }
-    
-    private FeatureDefinition graphemeFeatureDef(Set<String> phChains) throws IOException {
-        
-        String lineBreak = System.getProperty("line.separator");
-        
-        StringBuilder fdString = new StringBuilder("ByteValuedFeatureProcessors");
-        fdString.append(lineBreak);
-        
-        // add attribute features
-        for (int att = 1; att <= context*2 + 1; att++){
-            fdString.append("att").append(att);
+		Set<String> phChains = new HashSet<String>();
 
-            for (String gr :  this.graphemeSet) {
-                fdString.append(" ").append(gr);
-            }
-            fdString.append(lineBreak);
-        }
-        fdString.append("ShortValuedFeatureProcessors").append(lineBreak);
-        
-        // add class features
-        fdString.append(TrainedLTS.PREDICTED_STRING_FEATURENAME);
-        
-        for (String ph :  phChains){
-            fdString.append(" ").append(ph);
-        }
-        
-        fdString.append(lineBreak);
-        
-        fdString.append("ContinuousFeatureProcessors").append(lineBreak);
-        
-        BufferedReader featureReader = new BufferedReader(new StringReader(fdString.toString()));
-        
-        return new FeatureDefinition(featureReader,false);        
-    }
-    
-    /**
-     * 
-     * reads in a lexicon in text format, lines are of the kind:
-     * 
-     * graphemechain | phonechain | otherinformation
-     * 
-     * Stress is optionally preserved, marking the first vowel of a stressed
-     * syllable with "1".
-     * 
-     * @param lexicon reader with lines of lexicon
-     * @param splitPattern a regular expression used for identifying the field separator in each line.
-     * @throws IOException
-     */
-    public void readLexicon(BufferedReader lexicon, String splitPattern) throws IOException{
-                
-        String line;
-        
-        while ((line = lexicon.readLine()) != null){
-            String[] lineParts = line.trim().split(splitPattern);
-            String graphStr = lineParts[0];
-            if (convertToLowercase) graphStr = graphStr.toLowerCase(phSet.getLocale());
-            graphStr = graphStr.replaceAll("['-.]", "");
+		// for every alignment pair collect counts
+		for (int i = 0; i < this.inSplit.size(); i++) {
 
-            // remove all secondary stress markers
-            String phonStr = lineParts[1].replaceAll(",", "");
-            String[] syllables = phonStr.split("-");
-            List<String> separatedPhones = new ArrayList<String>();
-            List<String> separatedGraphemes = new ArrayList<String>();
-            String currPh;
-            for (String syl : syllables) {
-                boolean stress = false;
-                if (syl.startsWith("'")){
-                    syl = syl.substring(1);
-                    stress = true;
-                }
-                for (Allophone ph : phSet.splitIntoAllophones(syl)) {
-                    currPh = ph.name();
-                    if (stress && considerStress && ph.isVowel()) {
-                        currPh += "1";
-                        stress = false;
-                    }
-                    separatedPhones.add(currPh);
-                }// ... for each allophone
-            }
-            
-            for ( int i = 0 ; i < graphStr.length() ; i++ ) {
-                this.graphemeSet.add(graphStr.substring(i, i+1));
-                separatedGraphemes.add(graphStr.substring(i, i+1));
-            }
-            this.addAlreadySplit(separatedGraphemes, separatedPhones);
-        }
-        // Need one entry for the "null" grapheme, which maps to the empty string:
-        this.addAlreadySplit(new String[]{"null"}, new String[]{""});
-    }
-    
-    /**
-     * reads in a lexicon in text format, lines are of the kind:
-     * 
-     * graphemechain | phonechain | otherinformation
-     * 
-     * Stress is optionally preserved, marking the first vowel of a stressed
-     * syllable with "1".
-     * 
-     * @param lexicon
-     * @throws IOException
-     */
-    public void readLexicon(HashMap<String, String> lexicon) {
-        
-        Iterator<String> it = lexicon.keySet().iterator();
-        while (it.hasNext()){
-            String graphStr = it.next(); 
-            
-            // remove all secondary stress markers
-            String phonStr = lexicon.get(graphStr).replaceAll(",", "");
-            if (convertToLowercase) graphStr = graphStr.toLowerCase(phSet.getLocale());
-            graphStr = graphStr.replaceAll("['-.]", "");
+			StringPair[] alignment = this.getAlignment(i);
 
-            String[] syllables = phonStr.split("-");
-            List<String> separatedPhones = new ArrayList<String>();
-            List<String> separatedGraphemes = new ArrayList<String>();
-            String currPh;
-            for (String syl : syllables) {
-                boolean stress = false;
-                if (syl.startsWith("'")){
-                    syl = syl.substring(1);
-                    stress = true;
-                }
-                for (Allophone ph : phSet.splitIntoAllophones(syl)) {
-                    currPh = ph.name();
-                    if (stress && considerStress && ph.isVowel()) {
-                        currPh += "1";
-                        stress = false;
-                    }
-                    separatedPhones.add(currPh);
-                }// ... for each allophone
-            }
-            
-            for ( int i = 0 ; i < graphStr.length() ; i++ ) {
-                this.graphemeSet.add(graphStr.substring(i, i+1));
-                separatedGraphemes.add(graphStr.substring(i, i+1));
-            }
-            this.addAlreadySplit(separatedGraphemes, separatedPhones);
-        }
-        // Need one entry for the "null" grapheme, which maps to the empty string:
-        this.addAlreadySplit(new String[]{"null"}, new String[]{""});
-    }
-    
-    public static void main(String[] args) throws IOException, MaryConfigurationException {
+			for (int inNr = 0; inNr < alignment.length; inNr++) {
 
-        
-        String  phFileLoc = "/Users/benjaminroth/Desktop/mary/english/phone-list-engba.xml";
+				// System.err.println(alignment[inNr]);
 
-        
-        // initialize trainer 
-        LTSTrainer tp = new LTSTrainer(AllophoneSet.getAllophoneSet(phFileLoc), true, true, 2);
+				// quotation signs needed to represent empty string
+				String outAlNr = "'" + alignment[inNr].getString2() + "'";
 
-        BufferedReader lexReader = new BufferedReader(
-                new InputStreamReader(
-                new FileInputStream(
-                        "/Users/benjaminroth/Desktop/mary/english/sampa-lexicon.txt"),"ISO-8859-1"));
-        
-        // read lexicon for training
-        tp.readLexicon(lexReader, "\\\\");
+				// TODO: don't consider alignments to more than three characters
+				if (outAlNr.length() > 5)
+					continue;
 
-        // make some alignment iterations
-        for ( int i = 0 ; i < 5 ; i++ ){
-            System.out.println("iteration " + i);
-            tp.alignIteration();
-            
-        }
-        
-        
-        CART st = tp.trainTree(100);
+				phChains.add(outAlNr);
+
+				// storing context and target
+				String[] datapoint = new String[2 * context + 2];
+
+				for (int ct = 0; ct < 2 * context + 1; ct++) {
+					int pos = inNr - context + ct;
+
+					if (pos >= 0 && pos < alignment.length) {
+						datapoint[ct] = alignment[pos].getString1();
+					} else {
+						datapoint[ct] = "null";
+					}
+
+				}
+
+				// set target
+				datapoint[2 * context + 1] = outAlNr;
+
+				// add datapoint
+				grapheme2align.get(alignment[inNr].getString1()).add(datapoint);
+			}
+		}
+
+		// for conversion need feature definition file
+		FeatureDefinition fd = this.graphemeFeatureDef(phChains);
+
+		int centerGrapheme = fd.getFeatureIndex("att" + (context + 1));
+
+		List<CART> stl = new ArrayList<CART>(fd.getNumberOfValues(centerGrapheme));
+
+		for (String gr : fd.getPossibleValues(centerGrapheme)) {
+			System.out.println("      Training decision tree for: " + gr);
+			logger.debug("      Training decision tree for: " + gr);
+
+			ArrayList<Attribute> attributeDeclarations = new ArrayList<Attribute>();
+
+			// attributes with values
+			for (int att = 1; att <= context * 2 + 1; att++) {
+
+				// ...collect possible values
+				ArrayList<String> attVals = new ArrayList<String>();
+
+				String featureName = "att" + att;
+
+				for (String usableGrapheme : fd.getPossibleValues(fd.getFeatureIndex(featureName))) {
+					attVals.add(usableGrapheme);
+				}
+
+				attributeDeclarations.add(new Attribute(featureName, attVals));
+			}
+
+			List<String[]> datapoints = grapheme2align.get(gr);
+
+			// maybe training is faster with targets limited to grapheme
+			Set<String> graphSpecPh = new HashSet<String>();
+			for (String[] dp : datapoints) {
+				graphSpecPh.add(dp[dp.length - 1]);
+			}
+
+			// targetattribute
+			// ...collect possible values
+			ArrayList<String> targetVals = new ArrayList<String>();
+			for (String phc : graphSpecPh) {// todo: use either fd of phChains
+				targetVals.add(phc);
+			}
+			attributeDeclarations.add(new Attribute(TrainedLTS.PREDICTED_STRING_FEATURENAME, targetVals));
+
+			// now, create the dataset adding the datapoints
+			Instances data = new Instances(gr, attributeDeclarations, 0);
+
+			// datapoints
+			for (String[] point : datapoints) {
+
+				Instance currInst = new DenseInstance(data.numAttributes());
+				currInst.setDataset(data);
+
+				for (int i = 0; i < point.length; i++) {
+
+					currInst.setValue(i, point[i]);
+				}
+
+				data.add(currInst);
+			}
+
+			// Make the last attribute be the class
+			data.setClassIndex(data.numAttributes() - 1);
+
+			// build the tree without using the J48 wrapper class
+			// standard parameters are:
+			// binary split selection with minimum x instances at the leaves, tree is pruned, confidenced value, subtree raising,
+			// cleanup, don't collapse
+			// Here is used a modifed version of C45PruneableClassifierTree that allow using Unary Classes (see Issue #51)
+			C45PruneableClassifierTree decisionTree;
+			try {
+				decisionTree = new C45PruneableClassifierTreeWithUnary(new BinC45ModelSelection(minLeafData, data, true), true,
+						0.25f, true, true, false);
+				decisionTree.buildClassifier(data);
+			} catch (Exception e) {
+				throw new RuntimeException("couldn't train decisiontree using weka: ", e);
+			}
+
+			CART maryTree = TreeConverter.c45toStringCART(decisionTree, fd, data);
+
+			stl.add(maryTree);
+		}
+
+		DecisionNode.ByteDecisionNode rootNode = new DecisionNode.ByteDecisionNode(centerGrapheme, stl.size(), fd);
+		for (CART st : stl) {
+			rootNode.addDaughter(st.getRootNode());
+		}
+
+		Properties props = new Properties();
+		props.setProperty("lowercase", String.valueOf(convertToLowercase));
+		props.setProperty("stress", String.valueOf(considerStress));
+		props.setProperty("context", String.valueOf(context));
+
+		CART bigTree = new CART(rootNode, fd, props);
+
+		return bigTree;
+	}
+
+	/**
+	 * 
+	 * Convenience method to save files to graph2phon.wagon and graph2phon.pfeats in a specified directory with UTF-8 encoding.
+	 * 
+	 * @param tree
+	 * @param saveTreePath
+	 * @throws IOException
+	 */
+	public void save(CART tree, String saveTreefile) throws IOException {
+		MaryCARTWriter mcw = new MaryCARTWriter();
+		mcw.dumpMaryCART(tree, saveTreefile);
+	}
+
+	private FeatureDefinition graphemeFeatureDef(Set<String> phChains) throws IOException {
+
+		String lineBreak = System.getProperty("line.separator");
+
+		StringBuilder fdString = new StringBuilder("ByteValuedFeatureProcessors");
+		fdString.append(lineBreak);
+
+		// add attribute features
+		for (int att = 1; att <= context * 2 + 1; att++) {
+			fdString.append("att").append(att);
+
+			for (String gr : this.graphemeSet) {
+				fdString.append(" ").append(gr);
+			}
+			fdString.append(lineBreak);
+		}
+		fdString.append("ShortValuedFeatureProcessors").append(lineBreak);
+
+		// add class features
+		fdString.append(TrainedLTS.PREDICTED_STRING_FEATURENAME);
+
+		for (String ph : phChains) {
+			fdString.append(" ").append(ph);
+		}
+
+		fdString.append(lineBreak);
+
+		fdString.append("ContinuousFeatureProcessors").append(lineBreak);
+
+		BufferedReader featureReader = new BufferedReader(new StringReader(fdString.toString()));
+
+		return new FeatureDefinition(featureReader, false);
+	}
+
+	/**
+	 * 
+	 * reads in a lexicon in text format, lines are of the kind:
+	 * 
+	 * graphemechain | phonechain | otherinformation
+	 * 
+	 * Stress is optionally preserved, marking the first vowel of a stressed syllable with "1".
+	 * 
+	 * @param lexicon
+	 *            reader with lines of lexicon
+	 * @param splitPattern
+	 *            a regular expression used for identifying the field separator in each line.
+	 * @throws IOException
+	 */
+	public void readLexicon(BufferedReader lexicon, String splitPattern) throws IOException {
+
+		String line;
+
+		while ((line = lexicon.readLine()) != null) {
+			String[] lineParts = line.trim().split(splitPattern);
+			String graphStr = lineParts[0];
+			if (convertToLowercase)
+				graphStr = graphStr.toLowerCase(phSet.getLocale());
+			graphStr = graphStr.replaceAll("['-.]", "");
+
+			// remove all secondary stress markers
+			String phonStr = lineParts[1].replaceAll(",", "");
+			String[] syllables = phonStr.split("-");
+			List<String> separatedPhones = new ArrayList<String>();
+			List<String> separatedGraphemes = new ArrayList<String>();
+			String currPh;
+			for (String syl : syllables) {
+				boolean stress = false;
+				if (syl.startsWith("'")) {
+					syl = syl.substring(1);
+					stress = true;
+				}
+				for (Allophone ph : phSet.splitIntoAllophones(syl)) {
+					currPh = ph.name();
+					if (stress && considerStress && ph.isVowel()) {
+						currPh += "1";
+						stress = false;
+					}
+					separatedPhones.add(currPh);
+				}// ... for each allophone
+			}
+
+			for (int i = 0; i < graphStr.length(); i++) {
+				this.graphemeSet.add(graphStr.substring(i, i + 1));
+				separatedGraphemes.add(graphStr.substring(i, i + 1));
+			}
+			this.addAlreadySplit(separatedGraphemes, separatedPhones);
+		}
+		// Need one entry for the "null" grapheme, which maps to the empty string:
+		this.addAlreadySplit(new String[] { "null" }, new String[] { "" });
+	}
+
+	/**
+	 * reads in a lexicon in text format, lines are of the kind:
+	 * 
+	 * graphemechain | phonechain | otherinformation
+	 * 
+	 * Stress is optionally preserved, marking the first vowel of a stressed syllable with "1".
+	 * 
+	 * @param lexicon
+	 * @throws IOException
+	 */
+	public void readLexicon(HashMap<String, String> lexicon) {
+
+		Iterator<String> it = lexicon.keySet().iterator();
+		while (it.hasNext()) {
+			String graphStr = it.next();
+
+			// remove all secondary stress markers
+			String phonStr = lexicon.get(graphStr).replaceAll(",", "");
+			if (convertToLowercase)
+				graphStr = graphStr.toLowerCase(phSet.getLocale());
+			graphStr = graphStr.replaceAll("['-.]", "");
+
+			String[] syllables = phonStr.split("-");
+			List<String> separatedPhones = new ArrayList<String>();
+			List<String> separatedGraphemes = new ArrayList<String>();
+			String currPh;
+			for (String syl : syllables) {
+				boolean stress = false;
+				if (syl.startsWith("'")) {
+					syl = syl.substring(1);
+					stress = true;
+				}
+				for (Allophone ph : phSet.splitIntoAllophones(syl)) {
+					currPh = ph.name();
+					if (stress && considerStress && ph.isVowel()) {
+						currPh += "1";
+						stress = false;
+					}
+					separatedPhones.add(currPh);
+				}// ... for each allophone
+			}
+
+			for (int i = 0; i < graphStr.length(); i++) {
+				this.graphemeSet.add(graphStr.substring(i, i + 1));
+				separatedGraphemes.add(graphStr.substring(i, i + 1));
+			}
+			this.addAlreadySplit(separatedGraphemes, separatedPhones);
+		}
+		// Need one entry for the "null" grapheme, which maps to the empty string:
+		this.addAlreadySplit(new String[] { "null" }, new String[] { "" });
+	}
+
+	public static void main(String[] args) throws IOException, MaryConfigurationException {
+
+		String phFileLoc = "/Users/benjaminroth/Desktop/mary/english/phone-list-engba.xml";
+
+		// initialize trainer
+		LTSTrainer tp = new LTSTrainer(AllophoneSet.getAllophoneSet(phFileLoc), true, true, 2);
+
+		BufferedReader lexReader = new BufferedReader(new InputStreamReader(new FileInputStream(
+				"/Users/benjaminroth/Desktop/mary/english/sampa-lexicon.txt"), "ISO-8859-1"));
+
+		// read lexicon for training
+		tp.readLexicon(lexReader, "\\\\");
+
+		// make some alignment iterations
+		for (int i = 0; i < 5; i++) {
+			System.out.println("iteration " + i);
+			tp.alignIteration();
+
+		}
+
+		CART st = tp.trainTree(100);
+
+		tp.save(st, "/Users/benjaminroth/Desktop/mary/english/trees/");
+
+	}
 
-        tp.save(st, "/Users/benjaminroth/Desktop/mary/english/trees/");
-     
-    }
-    
 }
-

marytts-builder/src/main/java/marytts/tools/newlanguage/LexiconCreator.java

@@ -50,281 +50,266 @@ import org.apache.log4j.Logger;
 import org.apache.log4j.PatternLayout;
 
 /**
- * The LexiconCreator is the base class for creating the files needed to run the
- * phonemiser component for a new language. From a list of phonetically transcribed
- * words, the class will create:
+ * The LexiconCreator is the base class for creating the files needed to run the phonemiser component for a new language. From a
+ * list of phonetically transcribed words, the class will create:
  * <ul>
  * <li>a lexicon file, efficiently stored as a Finite State Transducer;</li>
  * <li>a letter-to-sound prediction file, as a decision tree in MARY format.</li>
  * </ul>
  * 
  * The input file is expected to contain data in the following format:
- * <code>grapheme | ' a l - l o - p h o n e s | (optional-part-of-speech)</code>
- * Hereby, the allophones must correspond to a defined allophone set, given in the constructor.
- * The file's encoding is expected to be UTF-8.
- * Subclasses of LexiconCreator can override prepareLexicon() to provide data in this format.
+ * <code>grapheme | ' a l - l o - p h o n e s | (optional-part-of-speech)</code> Hereby, the allophones must correspond to a
+ * defined allophone set, given in the constructor. The file's encoding is expected to be UTF-8. Subclasses of LexiconCreator can
+ * override prepareLexicon() to provide data in this format.
+ * 
  * @see AllophoneSet
  * @author marc
  *
  */
-public class LexiconCreator
-{
-    protected Logger logger;
-    protected AllophoneSet allophoneSet;
-    protected String lexiconFilename;
-    protected String fstFilename;
-    protected String ltsFilename;
-    protected boolean convertToLowercase;
-    protected boolean predictStress;
-    protected int context;
-    
-    /**
-     * Initialise a new lexicon creator.
-     * Letter to sound rules built with this lexicon creator will convert graphemes to
-     * lowercase before prediction, using the locale given in the allophone set;
-     * letter-to-sound rules will also predict stress;
-     * a context of 2 characters to the left and to the right of the current character will
-     * be used as predictive features.
-     * @param allophoneSet this specifies the set of phonetic symbols that can be used in the lexicon, and
-     * provides the locale of the lexicon
-     * @param lexiconFilename where to find the plain-text lexicon
-     * @param fstFilename where to create the compressed lexicon FST file
-     * @param ltsFilename where to create the letter-to-sound prediction tree.
-     */
-    public LexiconCreator(AllophoneSet allophoneSet, String lexiconFilename, String fstFilename, String ltsFilename)
-    {
-        this(allophoneSet, lexiconFilename, fstFilename, ltsFilename, true, true, 2);
-    }
-    
-    /**
-     * Initialise a new lexicon creator.
-     * @param allophoneSet this specifies the set of phonetic symbols that can be used in the lexicon, and
-     * provides the locale of the lexicon
-     * @param lexiconFilename where to find the plain-text lexicon
-     * @param fstFilename where to create the compressed lexicon FST file
-     * @param ltsFilename where to create the letter-to-sound prediction tree.
-     * @param convertToLowercase if true, Letter to sound rules built with this lexicon creator will convert graphemes to
-     * lowercase before prediction, using the locale given in the allophone set.
-     * @param predictStress if true, letter-to-sound rules will predict stress.
-     * @param context the number of characters to the left and to the right of the current character will
-     * be used as predictive features.
-     */
-    public LexiconCreator(AllophoneSet allophoneSet, String lexiconFilename, String fstFilename, String ltsFilename,
-            boolean convertToLowercase, boolean predictStress, int context)
-    {
-        this.allophoneSet = allophoneSet;
-        this.lexiconFilename = lexiconFilename;
-        this.fstFilename = fstFilename;
-        this.ltsFilename = ltsFilename;
-        this.convertToLowercase = convertToLowercase;
-        this.predictStress = predictStress;
-        this.context = context;
-        this.logger = MaryUtils.getLogger("LexiconCreator");
-    }
+public class LexiconCreator {
+	protected Logger logger;
+	protected AllophoneSet allophoneSet;
+	protected String lexiconFilename;
+	protected String fstFilename;
+	protected String ltsFilename;
+	protected boolean convertToLowercase;
+	protected boolean predictStress;
+	protected int context;
 
-    /**
-     * This base implementation does nothing. Subclasses can override this method
-     * to prepare a lexicon in the expected format, which should then be found at
-     * lexiconFilename.
-     */
-    protected void prepareLexicon()
-    throws IOException
-    {
-    }
-    
-    protected void compileFST()
-    throws IOException
-    {
-        logger.info("Compressing into FST:");
-        logger.info(" - aligning graphemes and allophones...");
-        BufferedReader br = new BufferedReader(new InputStreamReader(new FileInputStream(lexiconFilename), "UTF-8"));
-        AlignerTrainer at = new AlignerTrainer(false, true);
-        at.readLexicon(br, "\\s*\\|\\s*");
-        br.close();
+	/**
+	 * Initialise a new lexicon creator. Letter to sound rules built with this lexicon creator will convert graphemes to lowercase
+	 * before prediction, using the locale given in the allophone set; letter-to-sound rules will also predict stress; a context
+	 * of 2 characters to the left and to the right of the current character will be used as predictive features.
+	 * 
+	 * @param allophoneSet
+	 *            this specifies the set of phonetic symbols that can be used in the lexicon, and provides the locale of the
+	 *            lexicon
+	 * @param lexiconFilename
+	 *            where to find the plain-text lexicon
+	 * @param fstFilename
+	 *            where to create the compressed lexicon FST file
+	 * @param ltsFilename
+	 *            where to create the letter-to-sound prediction tree.
+	 */
+	public LexiconCreator(AllophoneSet allophoneSet, String lexiconFilename, String fstFilename, String ltsFilename) {
+		this(allophoneSet, lexiconFilename, fstFilename, ltsFilename, true, true, 2);
+	}
 
-        // make some alignment iterations
-        for ( int i = 0 ; i < 4 ; i++ ){
-            logger.info("     iteration " + (i+1));
-            at.alignIteration();
-        }
-        logger.info(" - entering alignments in trie...");
-        TransducerTrie t = new TransducerTrie();
-        for (int i = 0, size = at.lexiconSize(); i<size; i++){
-            t.add(at.getAlignment(i));
-            t.add(at.getInfoAlignment(i));
-        }
-        logger.info(" - minimizing trie...");
-        t.computeMinimization();
-        logger.info(" - writing transducer to disk...");
-        File of = new File(fstFilename);
-        DataOutputStream os = new DataOutputStream(new BufferedOutputStream(new FileOutputStream(of)));
-        t.writeFST(os,"UTF-8");
-        os.flush();
-        os.close();
-    }
-    
-    protected void testFST()
-    throws IOException
-    {
-        List<String> testGraphemes = new ArrayList<String>();
-        List<String> testAllophones = new ArrayList<String>();
-        List<String> testPos = new ArrayList<String>();
-        int N = 100; // every N'th entry is put into tests...
-        loadTestWords(testGraphemes, testAllophones, testPos, N);
-        logger.info(" - looking up "+testGraphemes.size()+" test words...");
-        FSTLookup fst = new FSTLookup(fstFilename);
-        for (int i=0, max=testGraphemes.size(); i<max; i++) {
-            String key = testGraphemes.get(i);
-            String expected = testAllophones.get(i);
-            String[] result = fst.lookup(key);
-            if (testPos.get(i) != null) {
-                String key2 = key +testPos.get(i);
-                String[] result2 = fst.lookup(key2);
-                if (!expected.equals(result2[0]))
-                    logger.info("    "+key2+" -> "+ Arrays.toString(result2)+ " (expected: "+expected+")");
-                // in addition, expected should be one of the results of a lookup without pos
-                boolean found = false;
-                for (String r : result) {
-                    if (expected.equals(r)) {
-                        found = true;
-                        break;
-                    }
-                }
-                if (!found)
-                    logger.info("    "+key+" -> "+ Arrays.toString(result)+ " (expected: "+expected+")");
-            } else {
-                if (!expected.equals(result[0]))
-                    logger.info("    "+key+" -> "+ Arrays.toString(result)+ " (expected: "+expected+")");
-            }
-        }
-        logger.info("...done!\n");
-    }
+	/**
+	 * Initialise a new lexicon creator.
+	 * 
+	 * @param allophoneSet
+	 *            this specifies the set of phonetic symbols that can be used in the lexicon, and provides the locale of the
+	 *            lexicon
+	 * @param lexiconFilename
+	 *            where to find the plain-text lexicon
+	 * @param fstFilename
+	 *            where to create the compressed lexicon FST file
+	 * @param ltsFilename
+	 *            where to create the letter-to-sound prediction tree.
+	 * @param convertToLowercase
+	 *            if true, Letter to sound rules built with this lexicon creator will convert graphemes to lowercase before
+	 *            prediction, using the locale given in the allophone set.
+	 * @param predictStress
+	 *            if true, letter-to-sound rules will predict stress.
+	 * @param context
+	 *            the number of characters to the left and to the right of the current character will be used as predictive
+	 *            features.
+	 */
+	public LexiconCreator(AllophoneSet allophoneSet, String lexiconFilename, String fstFilename, String ltsFilename,
+			boolean convertToLowercase, boolean predictStress, int context) {
+		this.allophoneSet = allophoneSet;
+		this.lexiconFilename = lexiconFilename;
+		this.fstFilename = fstFilename;
+		this.ltsFilename = ltsFilename;
+		this.convertToLowercase = convertToLowercase;
+		this.predictStress = predictStress;
+		this.context = context;
+		this.logger = MaryUtils.getLogger("LexiconCreator");
+	}
 
-    private void loadTestWords(List<String> testGraphemes,
-            List<String> testAllophones, List<String> testPos, int N)
-            throws UnsupportedEncodingException, FileNotFoundException,
-            IOException {
-        int n = 0;
-        BufferedReader br = new BufferedReader(new InputStreamReader(new FileInputStream(lexiconFilename), "UTF-8"));
-        String line;
-        while ((line = br.readLine()) != null) {
-            String[] parts = line.split("\\s*\\|\\s*");
-            String graphemes = parts[0];
-            String allophones = parts[1];
-            String pos = (parts.length > 2 && parts[2].length()>0) ? parts[2] : null;
-            n++;
-            if (n == N) {
-                testGraphemes.add(graphemes);
-                testAllophones.add(allophones);
-                testPos.add(pos);
-                n = 0;
-            }
-        }
-    }
-    
-    protected void compileLTS()
-    throws IOException
-    {
-        logger.info("Training letter-to-sound rules...");
-        // initialize trainer 
-        LTSTrainer tp = new LTSTrainer(allophoneSet, convertToLowercase, predictStress, context);
-        BufferedReader br = new BufferedReader(new InputStreamReader(new FileInputStream(lexiconFilename), "UTF-8"));
-        
-        logger.info(" - reading lexicon...");
-        // read lexicon for training
-        tp.readLexicon(br, "\\s*\\|\\s*");
+	/**
+	 * This base implementation does nothing. Subclasses can override this method to prepare a lexicon in the expected format,
+	 * which should then be found at lexiconFilename.
+	 */
+	protected void prepareLexicon() throws IOException {
+	}
 
-        logger.info(" - aligning...");
-        // make some alignment iterations
-        for ( int i = 0 ; i < 5 ; i++ ){
-            logger.info("     iteration " + (i+1));
-            tp.alignIteration();
-            
-        }
-        logger.info(" - training decision tree...");
-        CART st = tp.trainTree(10);
-        logger.info(" - saving...");
-        // new MARY cart format:
-        MaryCARTWriter mcw = new MaryCARTWriter();
-        mcw.dumpMaryCART(st, ltsFilename);
-        
-        // Alternative ways of saving the CART would be:
-        // MARY cart text format:
-        //PrintWriter pw = new PrintWriter("lib/modules/en/us/lexicon/cmudict.lts.tree.txt", "UTF-8");
-        //mcw.toTextOut(st, pw);
-        //pw.close();
-        // old wagon cart, text and binary format:
-        //WagonCARTWriter wcw = new WagonCARTWriter();
-        //wcw.dumpWagonCART(st, "lib/modules/en/us/lexicon/cmudict.lts.wagontree.binary");
-        //pw = new PrintWriter("lib/modules/en/us/lexicon/cmudict.lts.wagontree.txt", "UTF-8");
-        //wcw.toTextOut(st, pw);
-        //pw.close();
-        // For all of these, it would also be necessary to  separately save the feature definition:
-        //pw = new PrintWriter("lib/modules/en/us/lexicon/cmudict.lts.pfeats", "UTF-8");
-        //st.getFeatureDefinition().writeTo(pw, false);
-        //pw.close();
-    
-        
-    }
-    
-    protected void testLTS()
-    throws IOException, MaryConfigurationException
-    {
-        List<String> testGraphemes = new ArrayList<String>();
-        List<String> testAllophones = new ArrayList<String>();
-        List<String> testPos = new ArrayList<String>();
-        int N = 100; // every N'th entry is put into tests...
-        loadTestWords(testGraphemes, testAllophones, testPos, N);
+	protected void compileFST() throws IOException {
+		logger.info("Compressing into FST:");
+		logger.info(" - aligning graphemes and allophones...");
+		BufferedReader br = new BufferedReader(new InputStreamReader(new FileInputStream(lexiconFilename), "UTF-8"));
+		AlignerTrainer at = new AlignerTrainer(false, true);
+		at.readLexicon(br, "\\s*\\|\\s*");
+		br.close();
 
-        logger.info(" - loading LTS rules...");
-        MaryCARTReader cartReader = new MaryCARTReader();
-        CART st = cartReader.load(ltsFilename);
-        TrainedLTS lts = new TrainedLTS(allophoneSet, st);
+		// make some alignment iterations
+		for (int i = 0; i < 4; i++) {
+			logger.info("     iteration " + (i + 1));
+			at.alignIteration();
+		}
+		logger.info(" - entering alignments in trie...");
+		TransducerTrie t = new TransducerTrie();
+		for (int i = 0, size = at.lexiconSize(); i < size; i++) {
+			t.add(at.getAlignment(i));
+			t.add(at.getInfoAlignment(i));
+		}
+		logger.info(" - minimizing trie...");
+		t.computeMinimization();
+		logger.info(" - writing transducer to disk...");
+		File of = new File(fstFilename);
+		DataOutputStream os = new DataOutputStream(new BufferedOutputStream(new FileOutputStream(of)));
+		t.writeFST(os, "UTF-8");
+		os.flush();
+		os.close();
+	}
 
-        logger.info(" - looking up "+testGraphemes.size()+" test words...");
-        int max = testGraphemes.size();
-        int correct = 0;
-        for (int i=0; i<max; i++) {
-            String key = testGraphemes.get(i);
-            String expected = testAllophones.get(i);
-            String result = lts.syllabify(lts.predictPronunciation(key));
-            if (!expected.equals(result))
-                logger.info("    "+key+" -> "+ result + " (expected: "+expected+")");
-            else 
-                correct++;
-        }
-        logger.info("   for "+correct+" out of "+max+" prediction is identical to lexicon entry.");
-        logger.info("...done!\n");
-    }
-    
-    
-    public void createLexicon()
-    throws Exception
-    {
-        prepareLexicon();
-        compileFST();
-        testFST();
-        System.gc();
-        compileLTS();
-        testLTS();
-    }
-    
-    
-    /**
-     * @param args
-     */
-    public static void main(String[] args) 
-    throws Exception
-    {
-        PatternLayout layout = new PatternLayout("%d %m\n");
-        BasicConfigurator.configure(new ConsoleAppender(layout));
-        AllophoneSet allophoneSet = AllophoneSet.getAllophoneSet(args[0]);
-        String lexiconFilename = args[1];
-        String fstFilename = args[2];
-        String ltsFilename = args[3];
-        LexiconCreator lc = new LexiconCreator(allophoneSet, lexiconFilename, fstFilename, ltsFilename);
-        lc.createLexicon();
-    }
+	protected void testFST() throws IOException {
+		List<String> testGraphemes = new ArrayList<String>();
+		List<String> testAllophones = new ArrayList<String>();
+		List<String> testPos = new ArrayList<String>();
+		int N = 100; // every N'th entry is put into tests...
+		loadTestWords(testGraphemes, testAllophones, testPos, N);
+		logger.info(" - looking up " + testGraphemes.size() + " test words...");
+		FSTLookup fst = new FSTLookup(fstFilename);
+		for (int i = 0, max = testGraphemes.size(); i < max; i++) {
+			String key = testGraphemes.get(i);
+			String expected = testAllophones.get(i);
+			String[] result = fst.lookup(key);
+			if (testPos.get(i) != null) {
+				String key2 = key + testPos.get(i);
+				String[] result2 = fst.lookup(key2);
+				if (!expected.equals(result2[0]))
+					logger.info("    " + key2 + " -> " + Arrays.toString(result2) + " (expected: " + expected + ")");
+				// in addition, expected should be one of the results of a lookup without pos
+				boolean found = false;
+				for (String r : result) {
+					if (expected.equals(r)) {
+						found = true;
+						break;
+					}
+				}
+				if (!found)
+					logger.info("    " + key + " -> " + Arrays.toString(result) + " (expected: " + expected + ")");
+			} else {
+				if (!expected.equals(result[0]))
+					logger.info("    " + key + " -> " + Arrays.toString(result) + " (expected: " + expected + ")");
+			}
+		}
+		logger.info("...done!\n");
+	}
+
+	private void loadTestWords(List<String> testGraphemes, List<String> testAllophones, List<String> testPos, int N)
+			throws UnsupportedEncodingException, FileNotFoundException, IOException {
+		int n = 0;
+		BufferedReader br = new BufferedReader(new InputStreamReader(new FileInputStream(lexiconFilename), "UTF-8"));
+		String line;
+		while ((line = br.readLine()) != null) {
+			String[] parts = line.split("\\s*\\|\\s*");
+			String graphemes = parts[0];
+			String allophones = parts[1];
+			String pos = (parts.length > 2 && parts[2].length() > 0) ? parts[2] : null;
+			n++;
+			if (n == N) {
+				testGraphemes.add(graphemes);
+				testAllophones.add(allophones);
+				testPos.add(pos);
+				n = 0;
+			}
+		}
+	}
+
+	protected void compileLTS() throws IOException {
+		logger.info("Training letter-to-sound rules...");
+		// initialize trainer
+		LTSTrainer tp = new LTSTrainer(allophoneSet, convertToLowercase, predictStress, context);
+		BufferedReader br = new BufferedReader(new InputStreamReader(new FileInputStream(lexiconFilename), "UTF-8"));
+
+		logger.info(" - reading lexicon...");
+		// read lexicon for training
+		tp.readLexicon(br, "\\s*\\|\\s*");
+
+		logger.info(" - aligning...");
+		// make some alignment iterations
+		for (int i = 0; i < 5; i++) {
+			logger.info("     iteration " + (i + 1));
+			tp.alignIteration();
+
+		}
+		logger.info(" - training decision tree...");
+		CART st = tp.trainTree(10);
+		logger.info(" - saving...");
+		// new MARY cart format:
+		MaryCARTWriter mcw = new MaryCARTWriter();
+		mcw.dumpMaryCART(st, ltsFilename);
+
+		// Alternative ways of saving the CART would be:
+		// MARY cart text format:
+		// PrintWriter pw = new PrintWriter("lib/modules/en/us/lexicon/cmudict.lts.tree.txt", "UTF-8");
+		// mcw.toTextOut(st, pw);
+		// pw.close();
+		// old wagon cart, text and binary format:
+		// WagonCARTWriter wcw = new WagonCARTWriter();
+		// wcw.dumpWagonCART(st, "lib/modules/en/us/lexicon/cmudict.lts.wagontree.binary");
+		// pw = new PrintWriter("lib/modules/en/us/lexicon/cmudict.lts.wagontree.txt", "UTF-8");
+		// wcw.toTextOut(st, pw);
+		// pw.close();
+		// For all of these, it would also be necessary to separately save the feature definition:
+		// pw = new PrintWriter("lib/modules/en/us/lexicon/cmudict.lts.pfeats", "UTF-8");
+		// st.getFeatureDefinition().writeTo(pw, false);
+		// pw.close();
+
+	}
+
+	protected void testLTS() throws IOException, MaryConfigurationException {
+		List<String> testGraphemes = new ArrayList<String>();
+		List<String> testAllophones = new ArrayList<String>();
+		List<String> testPos = new ArrayList<String>();
+		int N = 100; // every N'th entry is put into tests...
+		loadTestWords(testGraphemes, testAllophones, testPos, N);
+
+		logger.info(" - loading LTS rules...");
+		MaryCARTReader cartReader = new MaryCARTReader();
+		CART st = cartReader.load(ltsFilename);
+		TrainedLTS lts = new TrainedLTS(allophoneSet, st);
+
+		logger.info(" - looking up " + testGraphemes.size() + " test words...");
+		int max = testGraphemes.size();
+		int correct = 0;
+		for (int i = 0; i < max; i++) {
+			String key = testGraphemes.get(i);
+			String expected = testAllophones.get(i);
+			String result = lts.syllabify(lts.predictPronunciation(key));
+			if (!expected.equals(result))
+				logger.info("    " + key + " -> " + result + " (expected: " + expected + ")");
+			else
+				correct++;
+		}
+		logger.info("   for " + correct + " out of " + max + " prediction is identical to lexicon entry.");
+		logger.info("...done!\n");
+	}
+
+	public void createLexicon() throws Exception {
+		prepareLexicon();
+		compileFST();
+		testFST();
+		System.gc();
+		compileLTS();
+		testLTS();
+	}
+
+	/**
+	 * @param args
+	 */
+	public static void main(String[] args) throws Exception {
+		PatternLayout layout = new PatternLayout("%d %m\n");
+		BasicConfigurator.configure(new ConsoleAppender(layout));
+		AllophoneSet allophoneSet = AllophoneSet.getAllophoneSet(args[0]);
+		String lexiconFilename = args[1];
+		String fstFilename = args[2];
+		String ltsFilename = args[3];
+		LexiconCreator lc = new LexiconCreator(allophoneSet, lexiconFilename, fstFilename, ltsFilename);
+		lc.createLexicon();
+	}
 
 }
-

marytts-builder/src/main/java/marytts/tools/newlanguage/en_US/CMUDict2MaryFST.java

@@ -39,215 +39,196 @@ import org.apache.log4j.ConsoleAppender;
 import org.apache.log4j.PatternLayout;
 
 /**
- * This class does a one-time, offline conversion from the CMUDict in Festival format
- * (cmudict-0.4.scm and cmudict_extensions.scm) into MARY format.
- * Specifically, the following steps are performed:
+ * This class does a one-time, offline conversion from the CMUDict in Festival format (cmudict-0.4.scm and cmudict_extensions.scm)
+ * into MARY format. Specifically, the following steps are performed:
  * <ol>
  * <li>conversion to a text format without brackets, using '|' as the delimiter between three fields:
- * <code>graphemes | allophones | part-of-speech(optional)</code>
- * </li>
+ * <code>graphemes | allophones | part-of-speech(optional)</code></li>
  * <li>conversion of the phonetic alphabet used from MRPA to SAMPA</li>
  * <li>creation of a compact FST representing the lexicon</li>
  * <li>training of Letter-to-sound rules from the data</li>
  * </ol>
+ * 
  * @author marc
  *
  */
-public class CMUDict2MaryFST extends LexiconCreator
-{
+public class CMUDict2MaryFST extends LexiconCreator {
 	private static final String LEXPATH = "lib/modules/en/us/lexicon/";
-	
-    public CMUDict2MaryFST()
-    throws Exception
-    {
-        super(AllophoneSet.getAllophoneSet(LEXPATH + "allophones.en_US.xml"),
-                LEXPATH + "cmudictSampa.txt",
-                LEXPATH + "cmudict.fst",
-                LEXPATH + "cmudict.lts",
-                true, // convert to lowercase
-                true, // predict stress
-                3 // number of characters to the left and to the right to use for prediction
-                );
-    }
-    
-    @Override
-    protected void prepareLexicon()
-    throws IOException
-    {
-        File cmudict = new File(LEXPATH + "cmu/cmudict-0.4.scm");
-        if (!cmudict.exists())
-            throw new IllegalStateException("This program should be called from the MARY base directory.");
-        File extensions = new File(LEXPATH + "cmu/cmudict_extensions.scm");
-        File cmudictSampa = new File(lexiconFilename);
-        
-        // Convert to SAMPA text dictionary
-        logger.info("Converting dictionary to MARY text format...");
-        mrpa2sampa = new HashMap<String, String>();
-        fillSampaMap();
-        
-        BufferedReader br = new BufferedReader(new InputStreamReader(new FileInputStream(cmudict), "ASCII"));
-        PrintWriter toSampa = new PrintWriter(cmudictSampa, "UTF-8");
-        convertToSampa(br, toSampa);
-        br.close();
-        br = new BufferedReader(new InputStreamReader(new FileInputStream(extensions), "ASCII"));
-        convertToSampa(br, toSampa);
-        br.close();
-        toSampa.close();
-        logger.info("...done!\n");
 
-    }
-    
-    
-    
-    
-    private Map<String, String> mrpa2sampa;
+	public CMUDict2MaryFST() throws Exception {
+		super(AllophoneSet.getAllophoneSet(LEXPATH + "allophones.en_US.xml"), LEXPATH + "cmudictSampa.txt", LEXPATH
+				+ "cmudict.fst", LEXPATH + "cmudict.lts", true, // convert to lowercase
+				true, // predict stress
+				3 // number of characters to the left and to the right to use for prediction
+		);
+	}
 
-    private void fillSampaMap()
-    {
-        // Any phone inventory mappings?
-        String sampamapFilename = "lib/modules/en/synthesis/sampa2mrpa_en.map";
-        try {
-            BufferedReader br = new BufferedReader(new InputStreamReader(new FileInputStream(sampamapFilename), "UTF-8"));
-            String line;
-            while ((line = br.readLine()) != null) {
-                line = line.trim();
-                if (line.equals("") || line.startsWith("#")) {
-                    continue; // ignore empty and comment lines
-                }
-                try {
-                    addSampaMapEntry(line);
-                } catch (IllegalArgumentException iae) {
-                    throw new IllegalArgumentException("Ignoring invalid entry in sampa map file "+sampamapFilename, iae);
-                }
-            }
-        } catch (IOException ioe) {
-            throw new IllegalArgumentException("Cannot open file '"+sampamapFilename+"'", ioe);
-        }
-    }
+	@Override
+	protected void prepareLexicon() throws IOException {
+		File cmudict = new File(LEXPATH + "cmu/cmudict-0.4.scm");
+		if (!cmudict.exists())
+			throw new IllegalStateException("This program should be called from the MARY base directory.");
+		File extensions = new File(LEXPATH + "cmu/cmudict_extensions.scm");
+		File cmudictSampa = new File(lexiconFilename);
 
-    private void addSampaMapEntry(String entry) throws IllegalArgumentException
-    {
-        boolean s2v = false;
-        boolean v2s = false;
-        String[] parts = null;
-        // For one-to-many mappings, '+' can be used to group phone symbols.
-        // E.g., the line "EI->E:+I" would map "EI" to "E:" and "I" 
-        entry = entry.replace('+', ' ');
-        if (entry.indexOf("<->") != -1) {
-            parts = entry.split("<->");
-            s2v = true;
-            v2s = true;
-        } else if (entry.indexOf("->") != -1) {
-            parts = entry.split("->");
-            s2v = true;
-        } else if (entry.indexOf("<-") != -1) {
-            parts = entry.split("<-");
-            v2s = true;
-        }
-        if (parts == null || parts.length != 2) { // invalid entry
-            throw new IllegalArgumentException();
-        }
-        if (v2s) {
-            mrpa2sampa.put(parts[1].trim(), parts[0].trim());
-        }
-    }
+		// Convert to SAMPA text dictionary
+		logger.info("Converting dictionary to MARY text format...");
+		mrpa2sampa = new HashMap<String, String>();
+		fillSampaMap();
 
-    /** Converts a single phonetic symbol in MRPA representation
-     * representation into its equivalent in MARY sampa representation.
-     * @return the converted phone, or the input string if no known conversion exists.
-     */
-    private String mrpa2sampa(String voicePhoneme)
-    {
-        if (mrpa2sampa.containsKey(voicePhoneme))
-            return mrpa2sampa.get(voicePhoneme);
-        else
-            return voicePhoneme;
-    }
+		BufferedReader br = new BufferedReader(new InputStreamReader(new FileInputStream(cmudict), "ASCII"));
+		PrintWriter toSampa = new PrintWriter(cmudictSampa, "UTF-8");
+		convertToSampa(br, toSampa);
+		br.close();
+		br = new BufferedReader(new InputStreamReader(new FileInputStream(extensions), "ASCII"));
+		convertToSampa(br, toSampa);
+		br.close();
+		toSampa.close();
+		logger.info("...done!\n");
 
-    private String mrpaString2sampaString(String mrpaString)
-    {
-        StringTokenizer st = new StringTokenizer(mrpaString);
-        LinkedList<String> sampaList = new LinkedList<String>();
-        while (st.hasMoreTokens()) {
-            String mrpa = st.nextToken();
-            String sampa;
-            if (mrpa.endsWith("1")) {
-                sampa = mrpa2sampa(mrpa.substring(0, mrpa.length()-1))+"1";
-            } else if (mrpa.endsWith("0")) {
-                sampa = mrpa2sampa(mrpa.substring(0, mrpa.length()-1));
-            } else {
-                sampa = mrpa2sampa(mrpa);
-            }
-            sampaList.add(sampa);
-        }
-        new Syllabifier(allophoneSet).syllabify(sampaList);
-        StringBuilder sb = new StringBuilder();
-        for (String s : sampaList) {
-            if (sb.length() > 0) sb.append(" ");
-            sb.append(s);
-        }
-        return sb.toString();
-    }
-    
-    private void convertToSampa(BufferedReader br, PrintWriter toSampa)
-    throws IOException 
-    {
-        String line;
-        while ((line = br.readLine()) != null) {
-            line = line.trim();
-            // skip comments:
-            if (line.startsWith(";") || line.equals("")) continue;
-            // expected line format:
-            // ("acquirer" nil (ax k w ay1 er0 er0))
-            int firstQuote = line.indexOf('"');
-            if (!(firstQuote >= 0)) {
-                System.err.println("Skipping strange line (no first quote): "+line);
-            }
-            int secondQuote = line.indexOf('"', firstQuote+1);
-            if (!(secondQuote > firstQuote)) {
-                System.err.println("Skipping strange line (no second quote): "+line);
-            }
-            int firstSpace = secondQuote+1;
-            if (!(line.charAt(firstSpace) == ' ')) {
-                System.err.println("Skipping strange line (no first space): "+line);
-            }
-            int secondSpace = line.indexOf(' ', firstSpace+1);
-            if (!(secondSpace > firstSpace)) {
-                System.err.println("Skipping strange line (no second space): "+line);
-            }
-            int firstBracket = secondSpace+1;
-            if (!(line.charAt(firstBracket) == '(')) {
-                System.err.println("Skipping strange line (no first bracket): "+line);
-            }
-            int secondBracket = line.indexOf(')', firstBracket+1);
-            if (!(secondBracket > firstBracket)) {
-                System.err.println("Skipping strange line (no second bracket): "+line);
-            }
-            String graphemes = line.substring(firstQuote+1, secondQuote);
-            String pos = line.substring(firstSpace+1, secondSpace);
-            if (pos.equals("nil")) pos = "";
-            else pos = "("+pos+")";
-            String allophones = line.substring(firstBracket+1, secondBracket);
-            String sampaString = mrpaString2sampaString(allophones);
-            toSampa.println(graphemes + " | " + sampaString + " | "+pos);
-        }
-    }
+	}
 
+	private Map<String, String> mrpa2sampa;
 
-    
-    /**
-     * @param args
-     */
-    public static void main(String[] args)
-    throws Exception
-    {
-        PatternLayout layout = new PatternLayout("%d %m\n");
-        BasicConfigurator.configure(new ConsoleAppender(layout));
+	private void fillSampaMap() {
+		// Any phone inventory mappings?
+		String sampamapFilename = "lib/modules/en/synthesis/sampa2mrpa_en.map";
+		try {
+			BufferedReader br = new BufferedReader(new InputStreamReader(new FileInputStream(sampamapFilename), "UTF-8"));
+			String line;
+			while ((line = br.readLine()) != null) {
+				line = line.trim();
+				if (line.equals("") || line.startsWith("#")) {
+					continue; // ignore empty and comment lines
+				}
+				try {
+					addSampaMapEntry(line);
+				} catch (IllegalArgumentException iae) {
+					throw new IllegalArgumentException("Ignoring invalid entry in sampa map file " + sampamapFilename, iae);
+				}
+			}
+		} catch (IOException ioe) {
+			throw new IllegalArgumentException("Cannot open file '" + sampamapFilename + "'", ioe);
+		}
+	}
 
-        CMUDict2MaryFST c2m = new CMUDict2MaryFST();
-        c2m.createLexicon();
-    }
+	private void addSampaMapEntry(String entry) throws IllegalArgumentException {
+		boolean s2v = false;
+		boolean v2s = false;
+		String[] parts = null;
+		// For one-to-many mappings, '+' can be used to group phone symbols.
+		// E.g., the line "EI->E:+I" would map "EI" to "E:" and "I"
+		entry = entry.replace('+', ' ');
+		if (entry.indexOf("<->") != -1) {
+			parts = entry.split("<->");
+			s2v = true;
+			v2s = true;
+		} else if (entry.indexOf("->") != -1) {
+			parts = entry.split("->");
+			s2v = true;
+		} else if (entry.indexOf("<-") != -1) {
+			parts = entry.split("<-");
+			v2s = true;
+		}
+		if (parts == null || parts.length != 2) { // invalid entry
+			throw new IllegalArgumentException();
+		}
+		if (v2s) {
+			mrpa2sampa.put(parts[1].trim(), parts[0].trim());
+		}
+	}
 
+	/**
+	 * Converts a single phonetic symbol in MRPA representation representation into its equivalent in MARY sampa representation.
+	 * 
+	 * @return the converted phone, or the input string if no known conversion exists.
+	 */
+	private String mrpa2sampa(String voicePhoneme) {
+		if (mrpa2sampa.containsKey(voicePhoneme))
+			return mrpa2sampa.get(voicePhoneme);
+		else
+			return voicePhoneme;
+	}
+
+	private String mrpaString2sampaString(String mrpaString) {
+		StringTokenizer st = new StringTokenizer(mrpaString);
+		LinkedList<String> sampaList = new LinkedList<String>();
+		while (st.hasMoreTokens()) {
+			String mrpa = st.nextToken();
+			String sampa;
+			if (mrpa.endsWith("1")) {
+				sampa = mrpa2sampa(mrpa.substring(0, mrpa.length() - 1)) + "1";
+			} else if (mrpa.endsWith("0")) {
+				sampa = mrpa2sampa(mrpa.substring(0, mrpa.length() - 1));
+			} else {
+				sampa = mrpa2sampa(mrpa);
+			}
+			sampaList.add(sampa);
+		}
+		new Syllabifier(allophoneSet).syllabify(sampaList);
+		StringBuilder sb = new StringBuilder();
+		for (String s : sampaList) {
+			if (sb.length() > 0)
+				sb.append(" ");
+			sb.append(s);
+		}
+		return sb.toString();
+	}
+
+	private void convertToSampa(BufferedReader br, PrintWriter toSampa) throws IOException {
+		String line;
+		while ((line = br.readLine()) != null) {
+			line = line.trim();
+			// skip comments:
+			if (line.startsWith(";") || line.equals(""))
+				continue;
+			// expected line format:
+			// ("acquirer" nil (ax k w ay1 er0 er0))
+			int firstQuote = line.indexOf('"');
+			if (!(firstQuote >= 0)) {
+				System.err.println("Skipping strange line (no first quote): " + line);
+			}
+			int secondQuote = line.indexOf('"', firstQuote + 1);
+			if (!(secondQuote > firstQuote)) {
+				System.err.println("Skipping strange line (no second quote): " + line);
+			}
+			int firstSpace = secondQuote + 1;
+			if (!(line.charAt(firstSpace) == ' ')) {
+				System.err.println("Skipping strange line (no first space): " + line);
+			}
+			int secondSpace = line.indexOf(' ', firstSpace + 1);
+			if (!(secondSpace > firstSpace)) {
+				System.err.println("Skipping strange line (no second space): " + line);
+			}
+			int firstBracket = secondSpace + 1;
+			if (!(line.charAt(firstBracket) == '(')) {
+				System.err.println("Skipping strange line (no first bracket): " + line);
+			}
+			int secondBracket = line.indexOf(')', firstBracket + 1);
+			if (!(secondBracket > firstBracket)) {
+				System.err.println("Skipping strange line (no second bracket): " + line);
+			}
+			String graphemes = line.substring(firstQuote + 1, secondQuote);
+			String pos = line.substring(firstSpace + 1, secondSpace);
+			if (pos.equals("nil"))
+				pos = "";
+			else
+				pos = "(" + pos + ")";
+			String allophones = line.substring(firstBracket + 1, secondBracket);
+			String sampaString = mrpaString2sampaString(allophones);
+			toSampa.println(graphemes + " | " + sampaString + " | " + pos);
+		}
+	}
+
+	/**
+	 * @param args
+	 */
+	public static void main(String[] args) throws Exception {
+		PatternLayout layout = new PatternLayout("%d %m\n");
+		BasicConfigurator.configure(new ConsoleAppender(layout));
+
+		CMUDict2MaryFST c2m = new CMUDict2MaryFST();
+		c2m.createLexicon();
+	}
 
 }
-

marytts-builder/src/main/java/marytts/tools/upgrade/Mary4To5VoiceConverter.java

@@ -42,76 +42,48 @@ import marytts.util.io.FileUtils;
 public class Mary4To5VoiceConverter {
 
 	private static final String EOL = IOUtils.LINE_SEPARATOR_WINDOWS;
-	
+
 	/**
-	 * The list of property suffixes which can read from a resource file. 
+	 * The list of property suffixes which can read from a resource file.
 	 */
-	private static final String[] PROPS_FOR_RESOURCES = new String[] {
-		"Ftd", "Ftf", "Ftm", "Fts", 
-		"Fmd", "Fmf", "Fmm", "Fms",
-		"Fgvf", "Fgvm", "Fgvs",
-		"FeaFile",
-		"trickyPhonesFile",
-		"Fif",
-		"targetCostWeights", "joinCostWeights", "cartFile",
-		"duration.data", "F0.data", "midF0.data", "rightF0.data",
-		"exampleTextFile",
-	};
-	
-	private static final String[] PROPS_FOR_FILESYSTEM = new String[] {
-		"featureFile", "joinCostFile", "unitsFile", "audioTimelineFile", "basenameTimeline",
-		"vocalization.unitfile", "vocalization.timeline", "vocalization.featurefile",
-		"vocalization.featureDefinitionFile", "vocalization.intonationfile",
-		"vocalization.mlsafeaturefile", "vocalization.mixedexcitationfilter",
-		"vocalization.intonation.featureDefinitionFile",
-		// 4.0 prosody carts:
-		"duration.cart", "duration.featuredefinition", 
-		"f0.cart.left", "f0.cart.mid", "f0.cart.right", "f0.featuredefinition"
-	};
-	
+	private static final String[] PROPS_FOR_RESOURCES = new String[] { "Ftd", "Ftf", "Ftm", "Fts", "Fmd", "Fmf", "Fmm", "Fms",
+			"Fgvf", "Fgvm", "Fgvs", "FeaFile", "trickyPhonesFile", "Fif", "targetCostWeights", "joinCostWeights", "cartFile",
+			"duration.data", "F0.data", "midF0.data", "rightF0.data", "exampleTextFile", };
+
+	private static final String[] PROPS_FOR_FILESYSTEM = new String[] { "featureFile", "joinCostFile", "unitsFile",
+			"audioTimelineFile", "basenameTimeline", "vocalization.unitfile", "vocalization.timeline",
+			"vocalization.featurefile", "vocalization.featureDefinitionFile", "vocalization.intonationfile",
+			"vocalization.mlsafeaturefile", "vocalization.mixedexcitationfilter",
+			"vocalization.intonation.featureDefinitionFile",
+			// 4.0 prosody carts:
+			"duration.cart", "duration.featuredefinition", "f0.cart.left", "f0.cart.mid", "f0.cart.right", "f0.featuredefinition" };
+
 	/**
 	 * The list of property suffixes which should be dropped when upgrading the config file.
 	 */
-	private static final String[] SUFFIXES_TO_DROP_UNITSEL = new String[] {
-	};
-	private static final String[] SUFFIXES_TO_DROP_HMM = new String[] {
-		"F0.data",
-		"duration.data",
-	};
-	private static final String[] PROPS_TO_DROP = new String[] {
-		"requires",
-		"provides",
-		"voice.version",
-		"en_US-voice.version",
-		"de-voice.version",
-		"it-voice.version",
-		"requires.marybase.version",
-		"requires.hmm.version",
-		"requires.en_US.version",
-		"requires.de.version",
-		"requires.it.version",
-		"requires.en_US.download",
-		"requires.de.download",
-		"requires.it.download",
-	};
-	
+	private static final String[] SUFFIXES_TO_DROP_UNITSEL = new String[] {};
+	private static final String[] SUFFIXES_TO_DROP_HMM = new String[] { "F0.data", "duration.data", };
+	private static final String[] PROPS_TO_DROP = new String[] { "requires", "provides", "voice.version", "en_US-voice.version",
+			"de-voice.version", "it-voice.version", "requires.marybase.version", "requires.hmm.version",
+			"requires.en_US.version", "requires.de.version", "requires.it.version", "requires.en_US.download",
+			"requires.de.download", "requires.it.download", };
+
 	private Logger logger;
 	private VoiceComponentDescription voiceDescription;
 	private File mary4Zip;
 	private Properties config;
 	private List<String> originalConfig;
-	
+
 	private File extractedDir;
 	private File compileDir;
 	private String domain;
 	private int samplingRate;
 	private File[] filesForResources;
 	private File[] filesForFilesystem;
-	
+
 	VoiceCompiler.MavenVoiceCompiler compiler;
-	
-	public Mary4To5VoiceConverter(
-			List<VoiceComponentDescription> voiceDescriptions, File voiceZip) {
+
+	public Mary4To5VoiceConverter(List<VoiceComponentDescription> voiceDescriptions, File voiceZip) {
 		voiceDescription = null;
 		mary4Zip = voiceZip;
 		for (VoiceComponentDescription d : voiceDescriptions) {
@@ -121,69 +93,69 @@ public class Mary4To5VoiceConverter {
 			}
 		}
 		if (voiceDescription == null) {
-			throw new IllegalArgumentException("No matching voice description for file "+mary4Zip.getName());
+			throw new IllegalArgumentException("No matching voice description for file " + mary4Zip.getName());
 		}
 		if (!MaryUtils.isLog4jConfigured()) {
 			BasicConfigurator.configure();
 		}
 		logger = Logger.getLogger(this.getClass());
-		logger.info(voiceDescription.getName()+" "+voiceDescription.getVersion()+" ("+voiceDescription.getLocale()+" "+voiceDescription.getGender()+")");
+		logger.info(voiceDescription.getName() + " " + voiceDescription.getVersion() + " (" + voiceDescription.getLocale() + " "
+				+ voiceDescription.getGender() + ")");
 	}
-	
+
 	private void convert() throws Exception {
 		logger.info("converting...");
 		File rootDir = mary4Zip.getParentFile();
-		extractedDir = new File(rootDir, voiceDescription.getName()+"-"+voiceDescription.getVersion());
-		logger.debug("... extracting archive to "+extractedDir.getPath());
+		extractedDir = new File(rootDir, voiceDescription.getName() + "-" + voiceDescription.getVersion());
+		logger.debug("... extracting archive to " + extractedDir.getPath());
 		if (extractedDir.exists()) {
-			logger.debug("Folder "+extractedDir.getPath()+" exists, trying to delete...");
+			logger.debug("Folder " + extractedDir.getPath() + " exists, trying to delete...");
 			extractedDir.delete();
 		}
 		FileUtils.unzipArchive(mary4Zip, extractedDir);
-		
-		
+
 		loadConfig(findConfigFile());
-		
-		compileDir = new File(rootDir, voiceDescription.getName()+"-"+Version.specificationVersion()+"-maven");
-		
-		domain = config.getProperty(getPropertyPrefix()+"domain");
-		samplingRate = Integer.parseInt(config.getProperty(getPropertyPrefix()+"samplingRate"));
-		
+
+		compileDir = new File(rootDir, voiceDescription.getName() + "-" + Version.specificationVersion() + "-maven");
+
+		domain = config.getProperty(getPropertyPrefix() + "domain");
+		samplingRate = Integer.parseInt(config.getProperty(getPropertyPrefix() + "samplingRate"));
+
 		filesForResources = getFilesForResources();
-		filesForFilesystem = getFilesForFilesystem();		
+		filesForFilesystem = getFilesForFilesystem();
 		Map<String, String> extraVariablesToSubstitute = null;
-		
-		compiler = new VoiceCompiler.MavenVoiceCompiler(compileDir, getVoiceName(), Version.specificationVersion(), 
-				voiceDescription.getLocale(), voiceDescription.getGender(), domain, samplingRate, isUnitSelectionVoice(), filesForResources, 
-				filesForFilesystem, extraVariablesToSubstitute);
+
+		compiler = new VoiceCompiler.MavenVoiceCompiler(compileDir, getVoiceName(), Version.specificationVersion(),
+				voiceDescription.getLocale(), voiceDescription.getGender(), domain, samplingRate, isUnitSelectionVoice(),
+				filesForResources, filesForFilesystem, extraVariablesToSubstitute);
 
 		logger.debug("Creating directories");
 		compiler.createDirectories();
-		
+
 		logger.debug("Copying template files");
 		compiler.copyTemplateFiles();
 
 		updateConfig(compiler.getPackageName());
 		saveConfig(compiler.getConfigFile());
-				
-		logger.debug("Copying voice files");		
+
+		logger.debug("Copying voice files");
 		compiler.copyVoiceFiles();
-		
-		if(!isUnitSelectionVoice()){
-		  logger.debug("Converting HMM PDF files from Mary 4.0 to Mary 5.0 format");
-		  convertMary4ToMary5HmmPdfFiles(compiler.getMainResourcesDir());
-		}
-				
-		logger.debug("Compiling with Maven");
-		compiler.compileWithMaven();
-		
-		String convertedZipFilename = getFilenamePrefix()+".zip";
-		File convertedZipFile = new File(compileDir+"/target/"+convertedZipFilename);
-		if (!convertedZipFile.exists()) {
-			throw new IOException("Maven should have created file "+convertedZipFile.getAbsolutePath()+" but file does not exist.");
+
+		if (!isUnitSelectionVoice()) {
+			logger.debug("Converting HMM PDF files from Mary 4.0 to Mary 5.0 format");
+			convertMary4ToMary5HmmPdfFiles(compiler.getMainResourcesDir());
+		}
+
+		logger.debug("Compiling with Maven");
+		compiler.compileWithMaven();
+
+		String convertedZipFilename = getFilenamePrefix() + ".zip";
+		File convertedZipFile = new File(compileDir + "/target/" + convertedZipFilename);
+		if (!convertedZipFile.exists()) {
+			throw new IOException("Maven should have created file " + convertedZipFile.getAbsolutePath()
+					+ " but file does not exist.");
 		}
 
-		
 		updateVoiceDescription(rootDir, convertedZipFile);
 
 		File finalZipFile = new File(rootDir, convertedZipFilename);
@@ -192,25 +164,25 @@ public class Mary4To5VoiceConverter {
 		}
 		boolean success = convertedZipFile.renameTo(finalZipFile);
 		if (!success) {
-			throw new IOException("Failure trying to move "+convertedZipFile.getAbsolutePath()+" to "+finalZipFile.getAbsolutePath());
+			throw new IOException("Failure trying to move " + convertedZipFile.getAbsolutePath() + " to "
+					+ finalZipFile.getAbsolutePath());
 		}
-		
+
 	}
 
 	protected void saveConfig(File configFile) throws IOException {
 		saveConfigToStream(new BufferedOutputStream(new FileOutputStream(configFile)));
 	}
 
-	
 	private boolean isEmpty(String line) {
 		return line.trim().isEmpty();
 	}
-	
+
 	private boolean isComment(String line) {
 		return line.trim().startsWith("#");
-		
+
 	}
-	
+
 	protected void saveConfigToStream(OutputStream out) throws IOException {
 		PrintWriter pw = new PrintWriter(new OutputStreamWriter(out, "UTF-8"));
 		StringBuilder comments = new StringBuilder();
@@ -247,24 +219,23 @@ public class Mary4To5VoiceConverter {
 	private File[] getFilesFromProperties(String[] propertySuffixes) throws IOException {
 		ArrayList<File> files = new ArrayList<File>();
 		for (String suffix : propertySuffixes) {
-			String key = getPropertyPrefix()+suffix;
+			String key = getPropertyPrefix() + suffix;
 			if (config.containsKey(key)) {
 				String value = config.getProperty(key);
 				if (!value.startsWith("MARY_BASE")) {
-					throw new IOException("Property '"+key+"' should hold a filename but the value is '"+value+"'");
+					throw new IOException("Property '" + key + "' should hold a filename but the value is '" + value + "'");
 				}
 				value = value.replace("MARY_BASE", extractedDir.getAbsolutePath());
 				File f = new File(value);
 				if (!f.exists()) {
-					throw new IOException("Config file refers to non-existing file '"+f.getAbsolutePath()+"'");
+					throw new IOException("Config file refers to non-existing file '" + f.getAbsolutePath() + "'");
 				}
 				files.add(f);
 			}
 		}
 		return files.toArray(new File[0]);
 	}
-	
-	
+
 	protected void updateConfig(String packageName) {
 		updatePropsForResources(packageName);
 		dropOutdatedProps();
@@ -273,13 +244,13 @@ public class Mary4To5VoiceConverter {
 
 	private void addNewProps() {
 		config.setProperty("locale", voiceDescription.getLocale().toString());
-		
+
 	}
 
 	private void dropOutdatedProps() {
 		String[] suffixesToDrop = isUnitSelectionVoice() ? SUFFIXES_TO_DROP_UNITSEL : SUFFIXES_TO_DROP_HMM;
 		for (String suffix : suffixesToDrop) {
-			String key = getPropertyPrefix()+suffix;
+			String key = getPropertyPrefix() + suffix;
 			config.remove(key);
 		}
 		for (String prop : PROPS_TO_DROP) {
@@ -289,9 +260,9 @@ public class Mary4To5VoiceConverter {
 
 	private void updatePropsForResources(String packageName) {
 		String oldPrefix = "MARY_BASE/lib/voices/(.*)/";
-		String newPrefix = "jar:/marytts/voice/"+packageName+"/";
+		String newPrefix = "jar:/marytts/voice/" + packageName + "/";
 		for (String suffix : PROPS_FOR_RESOURCES) {
-			String key = getPropertyPrefix()+suffix;
+			String key = getPropertyPrefix() + suffix;
 			if (config.containsKey(key)) {
 				String value = config.getProperty(key);
 				value = value.replaceFirst(oldPrefix, newPrefix);
@@ -303,11 +274,12 @@ public class Mary4To5VoiceConverter {
 	private File findConfigFile() throws IOException {
 		File confDir = new File(extractedDir, "conf");
 		if (!confDir.isDirectory()) {
-			throw new IOException("Expected directory "+confDir.getAbsolutePath()+" doesn't exist.");
+			throw new IOException("Expected directory " + confDir.getAbsolutePath() + " doesn't exist.");
 		}
 		File[] confFiles = confDir.listFiles();
 		if (confFiles.length != 1) {
-			throw new IOException("Conf directory "+confDir.getAbsolutePath()+" should contain exactly one config file but contains "+confFiles.length);
+			throw new IOException("Conf directory " + confDir.getAbsolutePath()
+					+ " should contain exactly one config file but contains " + confFiles.length);
 		}
 		return confFiles[0];
 	}
@@ -317,7 +289,7 @@ public class Mary4To5VoiceConverter {
 		InputStream bufferedConfigStream = IOUtils.toBufferedInputStream(configStream);
 		loadConfigFromStream(bufferedConfigStream);
 	}
-	
+
 	protected void loadConfigFromStream(InputStream in) throws IOException {
 		config = new Properties();
 		try {
@@ -328,9 +300,10 @@ public class Mary4To5VoiceConverter {
 			in.close();
 		}
 	}
-	
+
 	/**
 	 * Returns true for a unit selection voice, false for an HMM-based voice.
+	 * 
 	 * @return
 	 * @throw {@link UnsupportedOperationException} if the voice is neither a unit selection nor an HMM-based voice.
 	 */
@@ -340,44 +313,42 @@ public class Mary4To5VoiceConverter {
 		} else if (config.containsKey("hmm.voices.list")) {
 			return false;
 		} else {
-			throw new UnsupportedOperationException("The voice is neither a unit selection voice nor an HMM-based voice -- cannot convert to MARY 5 format.");
+			throw new UnsupportedOperationException(
+					"The voice is neither a unit selection voice nor an HMM-based voice -- cannot convert to MARY 5 format.");
 		}
 	}
-	
+
 	private String getVoiceNameFromConfig() {
 		if (isUnitSelectionVoice()) {
 			return config.getProperty("unitselection.voices.list");
 		}
 		return config.getProperty("hmm.voices.list");
 	}
-	
+
 	private String getVoiceNameFromVoiceDescription() {
 		return voiceDescription.getName();
 	}
-	
+
 	protected String getVoiceName() {
 		String voiceNameFromConfig = getVoiceNameFromConfig();
 		String voiceNameFromVoiceDescription = getVoiceNameFromVoiceDescription();
 		if (!voiceNameFromConfig.equals(voiceNameFromVoiceDescription)) {
-			logger.warn("Name discrepancy: component.xml says '"+voiceNameFromVoiceDescription+"', config file says '"+voiceNameFromConfig+"'");
+			logger.warn("Name discrepancy: component.xml says '" + voiceNameFromVoiceDescription + "', config file says '"
+					+ voiceNameFromConfig + "'");
 		}
 		return voiceNameFromVoiceDescription;
 	}
-	
+
 	protected String getPropertyPrefix() {
 		return "voice." + getVoiceNameFromConfig() + ".";
 	}
-	
 
 	private String getFilenamePrefix() {
-		return "voice-"+voiceDescription.getName()+"-"+Version.specificationVersion();
+		return "voice-" + voiceDescription.getName() + "-" + Version.specificationVersion();
 	}
-	
-	
 
-	private void updateVoiceDescription(File rootDir, File packageFile)
-			throws MalformedURLException, ParserConfigurationException,
-			MaryConfigurationException, IOException {
+	private void updateVoiceDescription(File rootDir, File packageFile) throws MalformedURLException,
+			ParserConfigurationException, MaryConfigurationException, IOException {
 		logger.debug("writing new voice description...");
 		voiceDescription.setVersion(Version.specificationVersion());
 		voiceDescription.setDependsVersion(Version.specificationVersion());
@@ -385,230 +356,229 @@ public class Mary4To5VoiceConverter {
 		voiceDescription.setPackageMD5Sum(computeMD5(packageFile));
 		voiceDescription.setPackageSize((int) packageFile.length());
 		voiceDescription.removeAllLocations();
-		voiceDescription.addLocation(URI.create("http://mary.dfki.de/download/"+Version.specificationVersion()+"/").toURL());
+		voiceDescription.addLocation(URI.create("http://mary.dfki.de/download/" + Version.specificationVersion() + "/").toURL());
 		Document doc = voiceDescription.createComponentXML();
 		File newVoiceDescriptionFile = new File(rootDir, getFilenamePrefix() + "-component.xml");
 		DomUtils.document2File(doc, newVoiceDescriptionFile);
-		logger.debug("... created "+newVoiceDescriptionFile.getPath());
+		logger.debug("... created " + newVoiceDescriptionFile.getPath());
 	}
 
-	
 	private String computeMD5(File packageFile) throws IOException {
 		return MD5.asHex(MD5.getHash(packageFile));
 	}
 
 	private void convertMary4ToMary5HmmPdfFiles(File mainResourcesDir) throws Exception {
-		
+
 		File list[] = mainResourcesDir.listFiles();
 		for (File f : list) {
 			// if mainResources dir contains f0.pdf mgc.pdf str.pdf and
-			if(f.getName().contains("dur.pdf") || f.getName().contains("lf0.pdf") || f.getName().contains("mgc.pdf") || f.getName().contains("str.pdf")) {
+			if (f.getName().contains("dur.pdf") || f.getName().contains("lf0.pdf") || f.getName().contains("mgc.pdf")
+					|| f.getName().contains("str.pdf")) {
 				logger.debug("converting file: " + f.getName());
 				convertPdfBinaryFile(f);
-			} 
-			// if mainResource contains gv-lf0-littend.pdf, gv-mgc-littend.pdf and gv-str-littend.pdf 
-			else if(f.getName().contains("gv-lf0-littend.pdf") || f.getName().contains("gv-mgc-littend.pdf") || f.getName().contains("gv-str-littend.pdf")) {
+			}
+			// if mainResource contains gv-lf0-littend.pdf, gv-mgc-littend.pdf and gv-str-littend.pdf
+			else if (f.getName().contains("gv-lf0-littend.pdf") || f.getName().contains("gv-mgc-littend.pdf")
+					|| f.getName().contains("gv-str-littend.pdf")) {
 				logger.debug("converting file: " + f.getName());
 				convertGvBinaryFile(f);
-			} 
+			}
 		}
-		
-		
-		
+
 	}
 
 	/**
 	 * Converts format from pdf Mary format 4 to Mary 5, the converted file will have the same input name
+	 * 
 	 * @param pdfInFile
 	 * @throws Exception
 	 */
 	public void convertPdfBinaryFile(File pdfInFile) throws Exception {
-	      int i,j,k,l;
-	      boolean lf0=false;
-	      
-	      String pdfInFileString = pdfInFile.getName(); 	      
-	      // the destination file name will be the same as the input file so
-	      String pdfOutFile = pdfInFile.getAbsolutePath();
-	      String path = pdfInFile.getParent();
-	      // I make a copy or the original file 
-	      FileUtils.copy(pdfInFile.getAbsolutePath(), path+"/tmp");
-	      pdfInFile = new File(path+"/tmp");
-	     	      
-	      DataInputStream dataIn;
-	      DataOutputStream dataOut;
+		int i, j, k, l;
+		boolean lf0 = false;
 
-	      dataIn = new DataInputStream (new BufferedInputStream(new FileInputStream(pdfInFile)));
-	      //numMSDFlag
-	      int numMSDFlag=0;
-	      //numStream
-	      int numStream=1; // 1 for mgc, str, mag or dur
-	      //vectorSize
-	      int vectorSize;
-	      //numDurPdf
-	      int numPdf[];
+		String pdfInFileString = pdfInFile.getName();
+		// the destination file name will be the same as the input file so
+		String pdfOutFile = pdfInFile.getAbsolutePath();
+		String path = pdfInFile.getParent();
+		// I make a copy or the original file
+		FileUtils.copy(pdfInFile.getAbsolutePath(), path + "/tmp");
+		pdfInFile = new File(path + "/tmp");
 
-	      logger.debug("Reading: from file " + pdfInFileString);
-	      float pdf[];  // pdf[vectorSize];
-	      float fval;
-	      int numState = 5;
+		DataInputStream dataIn;
+		DataOutputStream dataOut;
 
-	      //---------------------------------------------------------------------------------
-	      //------------ Read header --------------------------------------------------------
-	      //---------------------------------------------------------------------------------
-	      vectorSize = dataIn.readInt();
-	      if(pdfInFileString.contains("lf0.pdf")){
-	        numStream = vectorSize;
-	        vectorSize = 4; // vectorSize = 4 --> [1]:mean f0, [2]:var f0, [3]:voiced weight, [4]:unvoiced weight
-	        lf0 = true;
-	        numMSDFlag=1;
-	      } else if (pdfInFileString.contains("dur.pdf")){
-	        /* 2*nstate because the vector size for duration is the number of states */
-	        //pdf = new double[1][numDurPdf][1][2*numState];  // just one state and one stream
-	        numState=1;
-	        numStream=5;
-	      }
-	      logger.debug("vectorSize(r) = " + vectorSize + " numMSDFlag="+ numMSDFlag+ " numStream=" + numStream + " numState="+ numState);
+		dataIn = new DataInputStream(new BufferedInputStream(new FileInputStream(pdfInFile)));
+		// numMSDFlag
+		int numMSDFlag = 0;
+		// numStream
+		int numStream = 1; // 1 for mgc, str, mag or dur
+		// vectorSize
+		int vectorSize;
+		// numDurPdf
+		int numPdf[];
 
+		logger.debug("Reading: from file " + pdfInFileString);
+		float pdf[]; // pdf[vectorSize];
+		float fval;
+		int numState = 5;
 
-	      /* Now we need the number of pdf's for each state */
-	      numPdf = new int[numState];
-	      for(i=0; i< numState; i++){
-	         numPdf[i] = dataIn.readInt();
-	         logger.debug("loadPdfs(r): numPdf[state:"+ i + "]=" + numPdf[i]);
-	         if( numPdf[i] < 0 )
-	             throw new Exception("loadPdfs: #pdf at state " + i + " must be positive value.");
-	      }
-	      pdf = new float[2*vectorSize];
+		// ---------------------------------------------------------------------------------
+		// ------------ Read header --------------------------------------------------------
+		// ---------------------------------------------------------------------------------
+		vectorSize = dataIn.readInt();
+		if (pdfInFileString.contains("lf0.pdf")) {
+			numStream = vectorSize;
+			vectorSize = 4; // vectorSize = 4 --> [1]:mean f0, [2]:var f0, [3]:voiced weight, [4]:unvoiced weight
+			lf0 = true;
+			numMSDFlag = 1;
+		} else if (pdfInFileString.contains("dur.pdf")) {
+			/* 2*nstate because the vector size for duration is the number of states */
+			// pdf = new double[1][numDurPdf][1][2*numState]; // just one state and one stream
+			numState = 1;
+			numStream = 5;
+		}
+		logger.debug("vectorSize(r) = " + vectorSize + " numMSDFlag=" + numMSDFlag + " numStream=" + numStream + " numState="
+				+ numState);
 
-	      dataOut = new DataOutputStream (new BufferedOutputStream(new FileOutputStream(pdfOutFile)));
-	      // This is the format in version 2.0
-	      //numMSDFlag
-	      dataOut.writeInt(numMSDFlag);
-	      //numStream
-	      dataOut.writeInt(numStream);
-	      //vectorSize (for lf0 vectorsize is the same numStream)
-	      if(lf0)
-	        dataOut.writeInt(numStream);
-	      else
-	        dataOut.writeInt(vectorSize);
-	      //numPdf per state
-	      for(i=0; i< numState; i++){
-	        dataOut.writeInt(numPdf[i]);
-	      }
+		/* Now we need the number of pdf's for each state */
+		numPdf = new int[numState];
+		for (i = 0; i < numState; i++) {
+			numPdf[i] = dataIn.readInt();
+			logger.debug("loadPdfs(r): numPdf[state:" + i + "]=" + numPdf[i]);
+			if (numPdf[i] < 0)
+				throw new Exception("loadPdfs: #pdf at state " + i + " must be positive value.");
+		}
+		pdf = new float[2 * vectorSize];
 
-	      //---------------------------------------------------------------------------------
-	      //-------------- Now read the data ------------------------------------------------
-	      // in the old version the mean vector goes first and then the cov
-	      // in the new version the mean and cov elements of the vector are one after another
-	      //---------------------------------------------------------------------------------
-	       if(lf0){
-	        /* read  pdfs (mean, variance). (2*vectorSize because mean and diag variance */
-	        /* are allocated in only one vector. */
-	        for(i=0; i< numState; i++){
-	          for( j=0; j<numPdf[i]; j++){
-	              for( k=0; k<numStream; k++ ){
-	                for( l=0; l<vectorSize; l++) {
-	                  fval = dataIn.readFloat();
-	                  // NOTE: Here (hts_engine v1.04) the order seem to be the same as before
-	                  dataOut.writeFloat(fval);
-	                }
+		dataOut = new DataOutputStream(new BufferedOutputStream(new FileOutputStream(pdfOutFile)));
+		// This is the format in version 2.0
+		// numMSDFlag
+		dataOut.writeInt(numMSDFlag);
+		// numStream
+		dataOut.writeInt(numStream);
+		// vectorSize (for lf0 vectorsize is the same numStream)
+		if (lf0)
+			dataOut.writeInt(numStream);
+		else
+			dataOut.writeInt(vectorSize);
+		// numPdf per state
+		for (i = 0; i < numState; i++) {
+			dataOut.writeInt(numPdf[i]);
+		}
 
-	              }
-	          }
-	          //System.out.println("New pdf  j=" + j);
-	        }
-	       } else {
-	           /* read  pdfs (mean, variance). (2*vectorSize because mean and diag variance */
-	           /* are allocated in only one vector. */
-	           for(i=0; i< numState; i++){
-	             for( j=0; j<numPdf[i]; j++){
-	                 for( k=0; k<(2*vectorSize); k++ ){
-	                     pdf[k] = dataIn.readFloat();
-	                 }
-	                 for( k=0; k<vectorSize; k++ ){
-	                     dataOut.writeFloat(pdf[k]);
-	                     dataOut.writeFloat(pdf[k+vectorSize]);
-	                 }
-	             }
-	             //System.out.println("New pdf  j=" + j);
-	           }
+		// ---------------------------------------------------------------------------------
+		// -------------- Now read the data ------------------------------------------------
+		// in the old version the mean vector goes first and then the cov
+		// in the new version the mean and cov elements of the vector are one after another
+		// ---------------------------------------------------------------------------------
+		if (lf0) {
+			/* read pdfs (mean, variance). (2*vectorSize because mean and diag variance */
+			/* are allocated in only one vector. */
+			for (i = 0; i < numState; i++) {
+				for (j = 0; j < numPdf[i]; j++) {
+					for (k = 0; k < numStream; k++) {
+						for (l = 0; l < vectorSize; l++) {
+							fval = dataIn.readFloat();
+							// NOTE: Here (hts_engine v1.04) the order seem to be the same as before
+							dataOut.writeFloat(fval);
+						}
 
-	       }
-	      dataIn.close ();
-	      dataOut.close();
-	      pdfInFile.delete();
-	      logger.debug("Updated format in file " + pdfOutFile);
+					}
+				}
+				// System.out.println("New pdf  j=" + j);
+			}
+		} else {
+			/* read pdfs (mean, variance). (2*vectorSize because mean and diag variance */
+			/* are allocated in only one vector. */
+			for (i = 0; i < numState; i++) {
+				for (j = 0; j < numPdf[i]; j++) {
+					for (k = 0; k < (2 * vectorSize); k++) {
+						pdf[k] = dataIn.readFloat();
+					}
+					for (k = 0; k < vectorSize; k++) {
+						dataOut.writeFloat(pdf[k]);
+						dataOut.writeFloat(pdf[k + vectorSize]);
+					}
+				}
+				// System.out.println("New pdf  j=" + j);
+			}
 
+		}
+		dataIn.close();
+		dataOut.close();
+		pdfInFile.delete();
+		logger.debug("Updated format in file " + pdfOutFile);
 
-	  }
+	}
 
-      /**
-       * Converts file format from gv Mary format 4 to Mary 5, the converted file will have the same input name
-       * @param gvInFile
-       * @throws IOException
-       */
-	  public void convertGvBinaryFile(File gvInFile) throws IOException {
-	      int i;
-	      String gvInFileString = gvInFile.getName(); 
-	      // the destination file name will be the same as the input file so	      
-	      String gvOutFile = gvInFile.getAbsolutePath();
-	      String path = gvInFile.getParent();
-	      // I make a copy or the original file 
-	      FileUtils.copy(gvInFile.getAbsolutePath(), path+"/tmp");
-	      gvInFile = new File(path+"/tmp");
-	     	      
-	      DataInputStream dataIn;
-	      DataOutputStream dataOut;
-	      dataIn = new DataInputStream (new BufferedInputStream(new FileInputStream(gvInFile)));
+	/**
+	 * Converts file format from gv Mary format 4 to Mary 5, the converted file will have the same input name
+	 * 
+	 * @param gvInFile
+	 * @throws IOException
+	 */
+	public void convertGvBinaryFile(File gvInFile) throws IOException {
+		int i;
+		String gvInFileString = gvInFile.getName();
+		// the destination file name will be the same as the input file so
+		String gvOutFile = gvInFile.getAbsolutePath();
+		String path = gvInFile.getParent();
+		// I make a copy or the original file
+		FileUtils.copy(gvInFile.getAbsolutePath(), path + "/tmp");
+		gvInFile = new File(path + "/tmp");
 
-	      //int numMix = data_in.readShort();  /* --NOT USED -- first short is the number of mixtures in Gaussian model */
-	      int order = dataIn.readShort();     /* second short is the order of static vector */
-	      float gvmean[] = new float[order];  /* allocate memory of this size */
-	      float gvcov[] = new float[order];
-	      logger.debug("Reading from file " + gvInFileString + " order=" + order);
+		DataInputStream dataIn;
+		DataOutputStream dataOut;
+		dataIn = new DataInputStream(new BufferedInputStream(new FileInputStream(gvInFile)));
 
-	      for ( i = 0; i < order; i++){
-	          gvmean[i] = dataIn.readFloat();
-	          //System.out.format("gvmean[%d]=%f\n",i,gvmean[i]);
-	      }
-	      for ( i = 0; i < order; i++){
-	          gvcov[i] = dataIn.readFloat();
-	          //System.out.format("gvcov[%d]=%f\n",i,gvcov[i]);
-	      }
-	      dataIn.close ();
+		// int numMix = data_in.readShort(); /* --NOT USED -- first short is the number of mixtures in Gaussian model */
+		int order = dataIn.readShort(); /* second short is the order of static vector */
+		float gvmean[] = new float[order]; /* allocate memory of this size */
+		float gvcov[] = new float[order];
+		logger.debug("Reading from file " + gvInFileString + " order=" + order);
 
-	      dataOut = new DataOutputStream (new BufferedOutputStream(new FileOutputStream(gvOutFile)));
-	      /* This is the format in version 2.0*/
-	      //numMSDFlag
-	      dataOut.writeInt(0);
-	      //numStream
-	      dataOut.writeInt(1);
-	      //vectorSize
-	      dataOut.writeInt(order);
-	      //numDurPdf
-	      dataOut.writeInt(1);
+		for (i = 0; i < order; i++) {
+			gvmean[i] = dataIn.readFloat();
+			// System.out.format("gvmean[%d]=%f\n",i,gvmean[i]);
+		}
+		for (i = 0; i < order; i++) {
+			gvcov[i] = dataIn.readFloat();
+			// System.out.format("gvcov[%d]=%f\n",i,gvcov[i]);
+		}
+		dataIn.close();
 
-	      for (i = 0; i < order; i++){
-	        dataOut.writeFloat(gvmean[i]);
-	        dataOut.writeFloat(gvcov[i]);
-	      }
+		dataOut = new DataOutputStream(new BufferedOutputStream(new FileOutputStream(gvOutFile)));
+		/* This is the format in version 2.0 */
+		// numMSDFlag
+		dataOut.writeInt(0);
+		// numStream
+		dataOut.writeInt(1);
+		// vectorSize
+		dataOut.writeInt(order);
+		// numDurPdf
+		dataOut.writeInt(1);
 
-	      dataOut.close();
-	      gvInFile.delete();
-	      logger.debug("Updated format in file " + gvOutFile);
+		for (i = 0; i < order; i++) {
+			dataOut.writeFloat(gvmean[i]);
+			dataOut.writeFloat(gvcov[i]);
+		}
 
-	  }
+		dataOut.close();
+		gvInFile.delete();
+		logger.debug("Updated format in file " + gvOutFile);
 
+	}
 
 	public static void main(String[] args) {
 		if (args.length < 2) {
 			usage();
 			System.exit(1);
 		}
-		
+
 		File componentsFile = new File(args[0]);
 		if (!componentsFile.exists()) {
-			System.err.println("No component file: "+args[0]);
+			System.err.println("No component file: " + args[0]);
 			usage();
 			System.exit(1);
 		}
@@ -622,11 +592,11 @@ public class Mary4To5VoiceConverter {
 			usage();
 			System.exit(1);
 		}
-		
-		for (int i=1; i<args.length; i++) {
+
+		for (int i = 1; i < args.length; i++) {
 			File voiceZip = new File(args[i]);
 			if (!voiceZip.exists()) {
-				System.err.println("No such voice file: "+args[i]);
+				System.err.println("No such voice file: " + args[i]);
 				usage();
 				System.exit(1);
 			}
@@ -638,14 +608,12 @@ public class Mary4To5VoiceConverter {
 				System.exit(1);
 			}
 		}
-		
+
 	}
 
-
-
-
 	private static void usage() {
 		System.err.println("Usage:");
-		System.err.println("java "+Mary4To5VoiceConverter.class.getName()+" mary-components.xml mary-voice-file-4.3.0.zip [more voice files...]");
+		System.err.println("java " + Mary4To5VoiceConverter.class.getName()
+				+ " mary-components.xml mary-voice-file-4.3.0.zip [more voice files...]");
 	}
 }

marytts-builder/src/main/java/marytts/tools/voiceimport/AbstractTimelineMaker.java

@@ -33,172 +33,172 @@ import marytts.util.data.Datagram;
 
 public abstract class AbstractTimelineMaker extends VoiceImportComponent {
 
-    protected final String DATADIR = getName() + ".dataDir";
+	protected final String DATADIR = getName() + ".dataDir";
 
-    protected final String DATATIMELINE = getName() + ".dataTimeline";
+	protected final String DATATIMELINE = getName() + ".dataTimeline";
 
-    protected final String BASENAMETIMELINE = getName() + ".basenameTimeline";
+	protected final String BASENAMETIMELINE = getName() + ".basenameTimeline";
 
-    protected final String DATAEXT = getName() + ".dataExt";
+	protected final String DATAEXT = getName() + ".dataExt";
 
-    protected int percent;
+	protected int percent;
 
-    protected DatabaseLayout db;
+	protected DatabaseLayout db;
 
-    protected TimelineWriter dataTimeline;
+	protected TimelineWriter dataTimeline;
 
-    protected TimelineReader basenameTimeline;
+	protected TimelineReader basenameTimeline;
 
-    /**
-     * get the type of this {@link VoiceImportComponent}
-     * 
-     * @return the type
-     */
-    public abstract String getType();
+	/**
+	 * get the type of this {@link VoiceImportComponent}
+	 * 
+	 * @return the type
+	 */
+	public abstract String getType();
 
-    /**
-     * {@inheritDoc}
-     */
-    @Override
-    public SortedMap<String, String> getDefaultProps(DatabaseLayout theDB) {
-        db = theDB;
-        if (props == null) {
-            props = new TreeMap<String, String>();
-            props.put(DATADIR, db.getProp(db.ROOTDIR) + getType() + System.getProperty("file.separator"));
-            props.put(DATAEXT, "." + getType());
-            props.put(DATATIMELINE, db.getProp(db.FILEDIR) + "timeline_" + getType() + db.getProp(db.MARYEXT));
-            props.put(BASENAMETIMELINE, db.getProp(db.FILEDIR) + "timeline_basenames" + db.getProp(db.MARYEXT));
-        }
-        return props;
-    }
+	/**
+	 * {@inheritDoc}
+	 */
+	@Override
+	public SortedMap<String, String> getDefaultProps(DatabaseLayout theDB) {
+		db = theDB;
+		if (props == null) {
+			props = new TreeMap<String, String>();
+			props.put(DATADIR, db.getProp(db.ROOTDIR) + getType() + System.getProperty("file.separator"));
+			props.put(DATAEXT, "." + getType());
+			props.put(DATATIMELINE, db.getProp(db.FILEDIR) + "timeline_" + getType() + db.getProp(db.MARYEXT));
+			props.put(BASENAMETIMELINE, db.getProp(db.FILEDIR) + "timeline_basenames" + db.getProp(db.MARYEXT));
+		}
+		return props;
+	}
 
-    /**
-     * {@inheritDoc}
-     */
-    @Override
-    protected void setupHelp() {
-        props2Help = new TreeMap<String, String>();
-        props2Help.put(DATADIR, "Directory containing the " + getType() + " files");
-        props2Help.put(DATAEXT, "File extension of " + getType() + " files");
-        props2Help.put(DATATIMELINE, "File containing all " + getType() + ". Will be created by this module");
-        props2Help.put(BASENAMETIMELINE, "File containing all basenames.");
-    }
+	/**
+	 * {@inheritDoc}
+	 */
+	@Override
+	protected void setupHelp() {
+		props2Help = new TreeMap<String, String>();
+		props2Help.put(DATADIR, "Directory containing the " + getType() + " files");
+		props2Help.put(DATAEXT, "File extension of " + getType() + " files");
+		props2Help.put(DATATIMELINE, "File containing all " + getType() + ". Will be created by this module");
+		props2Help.put(BASENAMETIMELINE, "File containing all basenames.");
+	}
 
-    /**
-     * Read and concatenate a list of data files into a single timeline file.
-     * 
-     * @throws IOException
-     */
-    @Override
-    public boolean compute() throws IOException, MaryConfigurationException {
-        System.out.println("---- Importing data files");
-        System.out.println("Base directory: " + db.getProp(db.ROOTDIR));
+	/**
+	 * Read and concatenate a list of data files into a single timeline file.
+	 * 
+	 * @throws IOException
+	 */
+	@Override
+	public boolean compute() throws IOException, MaryConfigurationException {
+		System.out.println("---- Importing data files");
+		System.out.println("Base directory: " + db.getProp(db.ROOTDIR));
 
-        // read Files into List and check for readability early on:
-        List<File> files = new ArrayList<File>(bnl.getLength());
-        for (int b = 0; b < bnl.getLength(); b++) {
-            String basename = bnl.getName(b);
-            File file = new File(getProp(DATADIR) + basename + getProp(DATAEXT));
-            if (file.canRead()) {
-                files.add(file);
-            } else {
-                logger.warn("Ignoring unreadable file " + file);
-            }
-        }
-        // if List is still empty at this point, something is wrong; break:
-        if (files.size() == 0) {
-            return false;
-        }
+		// read Files into List and check for readability early on:
+		List<File> files = new ArrayList<File>(bnl.getLength());
+		for (int b = 0; b < bnl.getLength(); b++) {
+			String basename = bnl.getName(b);
+			File file = new File(getProp(DATADIR) + basename + getProp(DATAEXT));
+			if (file.canRead()) {
+				files.add(file);
+			} else {
+				logger.warn("Ignoring unreadable file " + file);
+			}
+		}
+		// if List is still empty at this point, something is wrong; break:
+		if (files.size() == 0) {
+			return false;
+		}
 
-        // load basename timeline:
-        String basenameTimelineFilename = getProp(BASENAMETIMELINE);
-        TimelineReader basenameTimelineReader = null;
-        basenameTimelineReader = new TimelineReader(basenameTimelineFilename);
+		// load basename timeline:
+		String basenameTimelineFilename = getProp(BASENAMETIMELINE);
+		TimelineReader basenameTimelineReader = null;
+		basenameTimelineReader = new TimelineReader(basenameTimelineFilename);
 
-        // process each file:
-        dataTimeline = null;
-        int p = 0; // progress counter
-        long basenameStart = 0;
-        for (File file : files) {
-            AbstractDataFile dataFile = loadDataFile(file);
+		// process each file:
+		dataTimeline = null;
+		int p = 0; // progress counter
+		long basenameStart = 0;
+		for (File file : files) {
+			AbstractDataFile dataFile = loadDataFile(file);
 
-            // first, make sure dataTimeLine is initialized (if not, init from dataFile):
-            if (dataTimeline == null) {
-                initializeDataTimeline(dataFile);
-            }
-            assert dataFile.sampleRate == dataTimeline.sampleRate;
+			// first, make sure dataTimeLine is initialized (if not, init from dataFile):
+			if (dataTimeline == null) {
+				initializeDataTimeline(dataFile);
+			}
+			assert dataFile.sampleRate == dataTimeline.sampleRate;
 
-            // get required duration from basename timeline:
-            Datagram basenameDatagram = basenameTimelineReader.getDatagram(basenameStart);
-            long basenameEnd = basenameStart + basenameDatagram.getDuration();
-            int requiredDuration = (int) (basenameEnd - basenameStart);
+			// get required duration from basename timeline:
+			Datagram basenameDatagram = basenameTimelineReader.getDatagram(basenameStart);
+			long basenameEnd = basenameStart + basenameDatagram.getDuration();
+			int requiredDuration = (int) (basenameEnd - basenameStart);
 
-            // get Datagrams for dataFile, enforcing requiredDuration, and feed them into the dataTimeline:
-            Datagram[] datagrams = dataFile.getDatagrams(requiredDuration);
-            dataTimeline.feed(datagrams, dataTimeline.getSampleRate());
+			// get Datagrams for dataFile, enforcing requiredDuration, and feed them into the dataTimeline:
+			Datagram[] datagrams = dataFile.getDatagrams(requiredDuration);
+			dataTimeline.feed(datagrams, dataTimeline.getSampleRate());
 
-            // set basenameStart to presumed start of next basename datagram:
-            basenameStart = basenameEnd + 1;
+			// set basenameStart to presumed start of next basename datagram:
+			basenameStart = basenameEnd + 1;
 
-            // progress info:
-            percent = 100 * p / files.size();
-            p++;
-            System.out.println(file.getName());
-        }
+			// progress info:
+			percent = 100 * p / files.size();
+			p++;
+			System.out.println(file.getName());
+		}
 
-        // finally, close the data timeline...
-        dataTimeline.close();
+		// finally, close the data timeline...
+		dataTimeline.close();
 
-        // ...and print some statistics:
-        System.out.println("---- data timeline result:");
-        System.out.println("Number of data files scanned: " + bnl.getLength());
-        System.out.println("Number of frames: [" + dataTimeline.getNumDatagrams() + "].");
-        int numIndex = dataTimeline.getIndex().getNumIdx();
-        System.out.println("Size of the index: [" + numIndex + "] (" + (dataTimeline.getIndex().getNumIdx() * 16)
-                + " bytes, i.e. " + String.format("%.2f", numIndex * 16.0 / 1048576.0) + " megs).");
-        System.out.println("---- data timeline done.");
+		// ...and print some statistics:
+		System.out.println("---- data timeline result:");
+		System.out.println("Number of data files scanned: " + bnl.getLength());
+		System.out.println("Number of frames: [" + dataTimeline.getNumDatagrams() + "].");
+		int numIndex = dataTimeline.getIndex().getNumIdx();
+		System.out.println("Size of the index: [" + numIndex + "] (" + (dataTimeline.getIndex().getNumIdx() * 16)
+				+ " bytes, i.e. " + String.format("%.2f", numIndex * 16.0 / 1048576.0) + " megs).");
+		System.out.println("---- data timeline done.");
 
-        return true;
-    }
+		return true;
+	}
 
-    /**
-     * load an {@link AbstractDataFile}
-     * 
-     * @param file
-     *            to load
-     * @return the AbstractDataFile
-     */
-    protected abstract AbstractDataFile loadDataFile(File file);
+	/**
+	 * load an {@link AbstractDataFile}
+	 * 
+	 * @param file
+	 *            to load
+	 * @return the AbstractDataFile
+	 */
+	protected abstract AbstractDataFile loadDataFile(File file);
 
-    /**
-     * initialize the data timeline, using one {@link AbstractDataFile} to provide the parameters
-     * 
-     * @param dataFile
-     *            to provide parameters
-     * @throws IOException
-     */
-    protected void initializeDataTimeline(AbstractDataFile dataFile) throws IOException {
-        String processingHeader = getProcessingHeader();
-        int sampleRate = dataFile.getSampleRate();
-        float frameSkip = dataFile.getFrameSkip();
-        dataTimeline = new TimelineWriter(getProp(DATATIMELINE), processingHeader, sampleRate, frameSkip);
-    }
+	/**
+	 * initialize the data timeline, using one {@link AbstractDataFile} to provide the parameters
+	 * 
+	 * @param dataFile
+	 *            to provide parameters
+	 * @throws IOException
+	 */
+	protected void initializeDataTimeline(AbstractDataFile dataFile) throws IOException {
+		String processingHeader = getProcessingHeader();
+		int sampleRate = dataFile.getSampleRate();
+		float frameSkip = dataFile.getFrameSkip();
+		dataTimeline = new TimelineWriter(getProp(DATATIMELINE), processingHeader, sampleRate, frameSkip);
+	}
 
-    /**
-     * generate a processing header for the {@link TimelineWriter}
-     * 
-     * @throws IOException
-     */
-    protected abstract String getProcessingHeader() throws IOException;
+	/**
+	 * generate a processing header for the {@link TimelineWriter}
+	 * 
+	 * @throws IOException
+	 */
+	protected abstract String getProcessingHeader() throws IOException;
 
-    /**
-     * Provide the progress of computation, in percent, or -1 if that feature is not implemented.
-     * 
-     * @return -1 if not implemented, or an integer between 0 and 100.
-     */
-    @Override
-    public int getProgress() {
-        return percent;
-    }
+	/**
+	 * Provide the progress of computation, in percent, or -1 if that feature is not implemented.
+	 * 
+	 * @return -1 if not implemented, or an integer between 0 and 100.
+	 */
+	@Override
+	public int getProgress() {
+		return percent;
+	}
 
 }

marytts-builder/src/main/java/marytts/tools/voiceimport/AcousticFeatureFileWriter.java

@@ -48,286 +48,266 @@ import marytts.util.data.MaryHeader;
 import marytts.util.math.ArrayUtils;
 import marytts.util.math.Polynomial;
 
+public class AcousticFeatureFileWriter extends VoiceImportComponent {
+	protected File maryDir;
+	protected FeatureFileReader feats;
+	protected FeatureDefinition inFeatureDefinition;
+	protected File outFeatureFile;
+	protected FeatureDefinition outFeatureDefinition;
+	protected UnitFileReader unitFileReader;
+	protected FeatureFileReader contours;
+	protected DatabaseLayout db = null;
+	protected int percent = 0;
 
-public class AcousticFeatureFileWriter extends VoiceImportComponent
-{
-    protected File maryDir;
-    protected FeatureFileReader feats;
-    protected FeatureDefinition inFeatureDefinition;
-    protected File outFeatureFile;
-    protected FeatureDefinition outFeatureDefinition;
-    protected UnitFileReader unitFileReader;
-    protected FeatureFileReader contours;
-    protected DatabaseLayout db = null;
-    protected int percent = 0;
-    
-    public final String UNITFILE = "AcousticFeatureFileWriter.unitFile";
-    public final String CONTOURFILE = "AcousticFeatureFileWriter.contourFile";
-    public final String FEATUREFILE = "AcousticFeatureFileWriter.featureFile";
-    public final String ACFEATUREFILE = "AcousticFeatureFileWriter.acFeatureFile";  
-    public final String ACFEATDEF = "AcousticFeatureFileWriter.acFeatDef";
-    
-    
-    public String getName(){
-        return "AcousticFeatureFileWriter";
-    }
-   
-    
-   public SortedMap<String, String> getDefaultProps(DatabaseLayout theDb)
-   {
-       this.db = theDb;
-       if (props == null){
-           props = new TreeMap<String, String>();
-           String fileDir = theDb.getProp(theDb.FILEDIR);
-           String maryExt = theDb.getProp(theDb.MARYEXT);
-           props.put(UNITFILE,fileDir+"halfphoneUnits"+maryExt);
-           props.put(CONTOURFILE,fileDir+"syllableF0Polynomials"+maryExt);
-           props.put(FEATUREFILE,fileDir+"halfphoneFeatures"+maryExt);
-           props.put(ACFEATUREFILE,fileDir+"halfphoneFeatures_ac"+maryExt);
-           props.put(ACFEATDEF,theDb.getProp(theDb.CONFIGDIR)+"halfphoneUnitFeatureDefinition_ac.txt");
-       }
-       return props;
-   }
-    
-   protected void setupHelp()
-   {
-       if (props2Help ==null) {
-           props2Help = new TreeMap<String, String>();
-           props2Help.put(UNITFILE,"file containing all halfphone units");
-           props2Help.put(CONTOURFILE,"file containing the polynomial contours for all syllables, indexed by phone features");
-           props2Help.put(FEATUREFILE,"file containing all halfphone units and their target cost features");
-           props2Help.put(ACFEATUREFILE,"file containing all halfphone units and their target cost features"
-								 +" plus the acoustic target cost features. Will be created by this module.");
-           props2Help.put(ACFEATDEF,"file containing the list of phone target cost features, their values and weights");
-       }
-   }
-  
-   @Override
-    public boolean compute() throws IOException, MaryConfigurationException
-    {
-        System.out.println("Acoustic feature file writer started.");
+	public final String UNITFILE = "AcousticFeatureFileWriter.unitFile";
+	public final String CONTOURFILE = "AcousticFeatureFileWriter.contourFile";
+	public final String FEATUREFILE = "AcousticFeatureFileWriter.featureFile";
+	public final String ACFEATUREFILE = "AcousticFeatureFileWriter.acFeatureFile";
+	public final String ACFEATDEF = "AcousticFeatureFileWriter.acFeatDef";
 
-        maryDir = new File( db.getProp(db.FILEDIR));
-        if (!maryDir.exists()) {
-            maryDir.mkdir();
-            System.out.println("Created the output directory [" + ( db.getProp(db.FILEDIR)) + "] to store the feature file." );
-        }
-        //System.out.println("A");
-        unitFileReader = new UnitFileReader(getProp(UNITFILE));
-        //System.out.println("B");
-        contours = new FeatureFileReader(getProp(CONTOURFILE));
-        //System.out.println("C");
-        
-        feats = new FeatureFileReader(getProp(FEATUREFILE));
-        inFeatureDefinition = feats.getFeatureDefinition();
-        StringWriter sw = new StringWriter();
-        PrintWriter pw = new PrintWriter(sw);
-        inFeatureDefinition.writeTo(pw, true);
-        // And now, append the two float features for duration and f0:
-        pw.println("100 linear | unit_duration");
-        pw.println("100 linear | unit_logf0");
-        pw.println("0 linear | unit_logf0delta");
-        pw.close();
-        String fd = sw.toString();
-        System.out.println("Generated the following feature definition:");
-        System.out.println(fd);
-        StringReader sr = new StringReader(fd);
-        BufferedReader br = new BufferedReader(sr);
-        outFeatureDefinition = new FeatureDefinition(br, true);
+	public String getName() {
+		return "AcousticFeatureFileWriter";
+	}
 
-        outFeatureFile = new File(getProp(ACFEATUREFILE));
-        DataOutputStream out = new DataOutputStream(new BufferedOutputStream(new FileOutputStream(outFeatureFile)));
-        writeHeaderTo(out);
-        writeUnitFeaturesTo(out);
-        out.close();
-        System.out.println("Number of processed units: " + unitFileReader.getNumberOfUnits() );
+	public SortedMap<String, String> getDefaultProps(DatabaseLayout theDb) {
+		this.db = theDb;
+		if (props == null) {
+			props = new TreeMap<String, String>();
+			String fileDir = theDb.getProp(theDb.FILEDIR);
+			String maryExt = theDb.getProp(theDb.MARYEXT);
+			props.put(UNITFILE, fileDir + "halfphoneUnits" + maryExt);
+			props.put(CONTOURFILE, fileDir + "syllableF0Polynomials" + maryExt);
+			props.put(FEATUREFILE, fileDir + "halfphoneFeatures" + maryExt);
+			props.put(ACFEATUREFILE, fileDir + "halfphoneFeatures_ac" + maryExt);
+			props.put(ACFEATDEF, theDb.getProp(theDb.CONFIGDIR) + "halfphoneUnitFeatureDefinition_ac.txt");
+		}
+		return props;
+	}
 
-        //make sure we have a feature definition with acoustic features
-        File featWeights = new File(getProp(ACFEATDEF));
-        if (!featWeights.exists()){
-            try{
-                PrintWriter featWeightsOut =
-                    new PrintWriter(
-                            new OutputStreamWriter(
-                                    new FileOutputStream(featWeights),"UTF-8"),true);
-                
-                
-                outFeatureDefinition.generateFeatureWeightsFile(featWeightsOut);                
-            } catch (Exception e) {
-                System.out.println("No halfphone feature weights ac file "
-                        +getProp(ACFEATDEF));
-                return false;
-            }
-        }
-        FeatureFileReader tester = FeatureFileReader.getFeatureFileReader(getProp(ACFEATUREFILE));
-        int unitsOnDisk = tester.getNumberOfUnits();
-        if (unitsOnDisk == unitFileReader.getNumberOfUnits()) {
-            System.out.println("Can read right number of units");
-            int r = new Random().nextInt(unitsOnDisk);
-            System.out.println("feature vector "+r+":");
-            System.out.println("Orig: "+feats.getFeatureVector(r).toString());
-            System.out.println("AC  : "+tester.getFeatureVector(r).toString());
-            return true;
-        } else {
-            System.out.println("Read wrong number of units: "+unitsOnDisk);
-            return false;
-        }
-    }
+	protected void setupHelp() {
+		if (props2Help == null) {
+			props2Help = new TreeMap<String, String>();
+			props2Help.put(UNITFILE, "file containing all halfphone units");
+			props2Help.put(CONTOURFILE, "file containing the polynomial contours for all syllables, indexed by phone features");
+			props2Help.put(FEATUREFILE, "file containing all halfphone units and their target cost features");
+			props2Help.put(ACFEATUREFILE, "file containing all halfphone units and their target cost features"
+					+ " plus the acoustic target cost features. Will be created by this module.");
+			props2Help.put(ACFEATDEF, "file containing the list of phone target cost features, their values and weights");
+		}
+	}
 
+	@Override
+	public boolean compute() throws IOException, MaryConfigurationException {
+		System.out.println("Acoustic feature file writer started.");
 
+		maryDir = new File(db.getProp(db.FILEDIR));
+		if (!maryDir.exists()) {
+			maryDir.mkdir();
+			System.out.println("Created the output directory [" + (db.getProp(db.FILEDIR)) + "] to store the feature file.");
+		}
+		// System.out.println("A");
+		unitFileReader = new UnitFileReader(getProp(UNITFILE));
+		// System.out.println("B");
+		contours = new FeatureFileReader(getProp(CONTOURFILE));
+		// System.out.println("C");
 
-    /**
-     * @param out
-     * @throws IOException
-     * @throws UnsupportedEncodingException
-     * @throws FileNotFoundException
-     */
-    protected void writeUnitFeaturesTo(DataOutput out) throws IOException, UnsupportedEncodingException, FileNotFoundException {
-        int numUnits = unitFileReader.getNumberOfUnits();
-        int unitSampleRate = unitFileReader.getSampleRate();
-        FeatureDefinition featureDefinition = feats.getFeatureDefinition();
-        int fiPhoneme = featureDefinition.getFeatureIndex("phone");
-        byte fvPhoneme_0 = featureDefinition.getFeatureValueAsByte(fiPhoneme, "0");
-        byte fvPhoneme_Silence = featureDefinition.getFeatureValueAsByte(fiPhoneme, "_");
-        int fiVowel = featureDefinition.getFeatureIndex("ph_vc");
-        byte fvVowel = featureDefinition.getFeatureValueAsByte(fiVowel, "+");
-        int fiLR = featureDefinition.getFeatureIndex("halfphone_lr");
-        byte fvLR_L = featureDefinition.getFeatureValueAsByte(fiLR, "L");
-        byte fvLR_R = featureDefinition.getFeatureValueAsByte(fiLR, "R");
-        int fiSylStart = featureDefinition.getFeatureIndex("segs_from_syl_start");
-        int fiSylEnd = featureDefinition.getFeatureIndex("segs_from_syl_end");
-        int iSylVowel = -1;
-        List<Float> unitDurs = new ArrayList<Float>();
-        
-        out.writeInt( numUnits );
-        System.out.println("Number of units : "+numUnits);
-        int iCurrent = 0;
-        for (int i=0; i<numUnits; i++) {
-            percent = 100*i/numUnits;
-            FeatureVector inFV = feats.getFeatureVector(i);
-            Unit u = unitFileReader.getUnit(i);
-            float dur = u.duration / (float) unitSampleRate;
+		feats = new FeatureFileReader(getProp(FEATUREFILE));
+		inFeatureDefinition = feats.getFeatureDefinition();
+		StringWriter sw = new StringWriter();
+		PrintWriter pw = new PrintWriter(sw);
+		inFeatureDefinition.writeTo(pw, true);
+		// And now, append the two float features for duration and f0:
+		pw.println("100 linear | unit_duration");
+		pw.println("100 linear | unit_logf0");
+		pw.println("0 linear | unit_logf0delta");
+		pw.close();
+		String fd = sw.toString();
+		System.out.println("Generated the following feature definition:");
+		System.out.println(fd);
+		StringReader sr = new StringReader(fd);
+		BufferedReader br = new BufferedReader(sr);
+		outFeatureDefinition = new FeatureDefinition(br, true);
 
-            // No syllable structure for edge and silence phone entries:
-            if (inFV.getByteFeature(fiPhoneme) == fvPhoneme_0
-                    || inFV.getByteFeature(fiPhoneme) == fvPhoneme_Silence) {
-                unitDurs.add(dur);
-                continue;
-            }
-            // Else, unit belongs to a syllable
-            if (inFV.getByteFeature(fiSylStart) == 0 && inFV.getByteFeature(fiLR) == fvLR_L) { // first segment in syllable
-                if (iCurrent < i) { // Something to output before this syllable
-                    assert i-iCurrent == unitDurs.size();
-                    writeFeatureVectors(out, iCurrent, iSylVowel, i-1, unitDurs);
-                }
-                unitDurs.clear();
-                iSylVowel = -1;
-                iCurrent = i;
-            }
-            
-            unitDurs.add(dur);
+		outFeatureFile = new File(getProp(ACFEATUREFILE));
+		DataOutputStream out = new DataOutputStream(new BufferedOutputStream(new FileOutputStream(outFeatureFile)));
+		writeHeaderTo(out);
+		writeUnitFeaturesTo(out);
+		out.close();
+		System.out.println("Number of processed units: " + unitFileReader.getNumberOfUnits());
 
-            if (inFV.getByteFeature(fiVowel) == fvVowel && iSylVowel == -1) { // the first vowel in the syllable
-                iSylVowel = i;
-            }
-            
-            if (inFV.getByteFeature(fiSylEnd) == 0 && inFV.getByteFeature(fiLR) == fvLR_R) { // last segment in syllable
-                writeFeatureVectors(out, iCurrent, iSylVowel, i, unitDurs);
-                iSylVowel = -1;
-                unitDurs.clear();
-                iCurrent = i+1;
-            }
-            
-        }
-        
-        assert numUnits-iCurrent == unitDurs.size();
-        writeFeatureVectors(out, iCurrent, iSylVowel, numUnits-1, unitDurs);
-    }
+		// make sure we have a feature definition with acoustic features
+		File featWeights = new File(getProp(ACFEATDEF));
+		if (!featWeights.exists()) {
+			try {
+				PrintWriter featWeightsOut = new PrintWriter(new OutputStreamWriter(new FileOutputStream(featWeights), "UTF-8"),
+						true);
 
+				outFeatureDefinition.generateFeatureWeightsFile(featWeightsOut);
+			} catch (Exception e) {
+				System.out.println("No halfphone feature weights ac file " + getProp(ACFEATDEF));
+				return false;
+			}
+		}
+		FeatureFileReader tester = FeatureFileReader.getFeatureFileReader(getProp(ACFEATUREFILE));
+		int unitsOnDisk = tester.getNumberOfUnits();
+		if (unitsOnDisk == unitFileReader.getNumberOfUnits()) {
+			System.out.println("Can read right number of units");
+			int r = new Random().nextInt(unitsOnDisk);
+			System.out.println("feature vector " + r + ":");
+			System.out.println("Orig: " + feats.getFeatureVector(r).toString());
+			System.out.println("AC  : " + tester.getFeatureVector(r).toString());
+			return true;
+		} else {
+			System.out.println("Read wrong number of units: " + unitsOnDisk);
+			return false;
+		}
+	}
 
-    private void writeFeatureVectors(DataOutput out, int iFirst,
-            int iVowel, int iLast, List<Float> unitDurs)
-            throws IOException {
-        float[] coeffs = null;
-        if (iVowel != -1) { // Syllable contains a vowel 
-            coeffs = contours.getFeatureVector(iVowel).getContinuousFeatures();
-            boolean isZero = true;
-            for (int c=0; c<coeffs.length; c++) {
-                if (coeffs[c] != 0) {
-                    isZero = false;
-                    break;
-                }
-            }
-            if (isZero) {
-                coeffs = null;
-            }
-        }
-        assert unitDurs.size() == iLast - iFirst + 1;
-        
-        float sylDur = 0;
-        for (int i=0, max=unitDurs.size(); i<max; i++) {
-            sylDur += unitDurs.get(i);
-        }
-        //System.out.println("Syl dur: "+sylDur+", "+unitDurs.size()+" units");
-        
-        float uStart = 0, uEnd = 0;
-        for (int i=0; iFirst+i<=iLast; i++) {
-            float logF0 = Float.NaN;
-            float logF0delta = Float.NaN;
-            if (coeffs != null && unitDurs.get(i) > 0) {
-                float relUStart = uStart / sylDur; // in [0, 1[
-                float relUEnd = (uStart+unitDurs.get(i)) / sylDur; // in [0, 1[
-                double[] predUnitContour = Polynomial.generatePolynomialValues(ArrayUtils.copyFloat2Double(coeffs), 10, relUStart, relUEnd);
-                //System.out.printf("From %.2f to %.2f:", relUStart, relUEnd);
-                //for (int k=0; k<predUnitContour.length; k++) System.out.printf(" %.2f", predUnitContour[k]);
-                // And fit a linear curve to this:
-                double[] unitCoeffs = Polynomial.fitPolynomial(predUnitContour, 1);
-                assert unitCoeffs.length == 2; // unitCoeffs[0] is the slope, unitCoeffs[1] the value at left end of interval.
-                // We need the f0 value in the middle of the unit:
-                logF0 = (float) (unitCoeffs[1] + 0.5 * unitCoeffs[0]);
-                logF0delta = (float) unitCoeffs[0];
-                //System.out.printf(" -- midF0=%.2f, slope=%.2f\n", logF0, logF0delta);
-            }
-            FeatureVector fv = feats.getFeatureVector(iFirst+i);
-            String line = fv.toString() + " " + unitDurs.get(i) + " " + logF0 + " " + logF0delta;
-            //System.out.println("fv: "+line);
-            FeatureVector outFV = outFeatureDefinition.toFeatureVector(0, line);
-            outFV.writeTo(out);
-            uStart += unitDurs.get(i);
-        }
-    }
+	/**
+	 * @param out
+	 * @throws IOException
+	 * @throws UnsupportedEncodingException
+	 * @throws FileNotFoundException
+	 */
+	protected void writeUnitFeaturesTo(DataOutput out) throws IOException, UnsupportedEncodingException, FileNotFoundException {
+		int numUnits = unitFileReader.getNumberOfUnits();
+		int unitSampleRate = unitFileReader.getSampleRate();
+		FeatureDefinition featureDefinition = feats.getFeatureDefinition();
+		int fiPhoneme = featureDefinition.getFeatureIndex("phone");
+		byte fvPhoneme_0 = featureDefinition.getFeatureValueAsByte(fiPhoneme, "0");
+		byte fvPhoneme_Silence = featureDefinition.getFeatureValueAsByte(fiPhoneme, "_");
+		int fiVowel = featureDefinition.getFeatureIndex("ph_vc");
+		byte fvVowel = featureDefinition.getFeatureValueAsByte(fiVowel, "+");
+		int fiLR = featureDefinition.getFeatureIndex("halfphone_lr");
+		byte fvLR_L = featureDefinition.getFeatureValueAsByte(fiLR, "L");
+		byte fvLR_R = featureDefinition.getFeatureValueAsByte(fiLR, "R");
+		int fiSylStart = featureDefinition.getFeatureIndex("segs_from_syl_start");
+		int fiSylEnd = featureDefinition.getFeatureIndex("segs_from_syl_end");
+		int iSylVowel = -1;
+		List<Float> unitDurs = new ArrayList<Float>();
 
-    /**
-     * Write the header of this feature file to the given DataOutput
-     * @param out
-     * @throws IOException
-     */
-    protected void writeHeaderTo(DataOutput out) throws IOException {
-        new MaryHeader(MaryHeader.UNITFEATS).writeTo(out);
-        outFeatureDefinition.writeBinaryTo(out);
-    }
+		out.writeInt(numUnits);
+		System.out.println("Number of units : " + numUnits);
+		int iCurrent = 0;
+		for (int i = 0; i < numUnits; i++) {
+			percent = 100 * i / numUnits;
+			FeatureVector inFV = feats.getFeatureVector(i);
+			Unit u = unitFileReader.getUnit(i);
+			float dur = u.duration / (float) unitSampleRate;
 
-    /**
-     * Provide the progress of computation, in percent, or -1 if
-     * that feature is not implemented.
-     * @return -1 if not implemented, or an integer between 0 and 100.
-     */
-    public int getProgress()
-    {
-        return percent;
-    }
+			// No syllable structure for edge and silence phone entries:
+			if (inFV.getByteFeature(fiPhoneme) == fvPhoneme_0 || inFV.getByteFeature(fiPhoneme) == fvPhoneme_Silence) {
+				unitDurs.add(dur);
+				continue;
+			}
+			// Else, unit belongs to a syllable
+			if (inFV.getByteFeature(fiSylStart) == 0 && inFV.getByteFeature(fiLR) == fvLR_L) { // first segment in syllable
+				if (iCurrent < i) { // Something to output before this syllable
+					assert i - iCurrent == unitDurs.size();
+					writeFeatureVectors(out, iCurrent, iSylVowel, i - 1, unitDurs);
+				}
+				unitDurs.clear();
+				iSylVowel = -1;
+				iCurrent = i;
+			}
 
-    /**
-     * @param args
-     */
-    public static void main(String[] args) throws Exception
-    {
-        AcousticFeatureFileWriter acfeatsWriter = 
-            new AcousticFeatureFileWriter();
-        DatabaseLayout db = new DatabaseLayout(acfeatsWriter);
-        acfeatsWriter.compute();
-    }
+			unitDurs.add(dur);
 
+			if (inFV.getByteFeature(fiVowel) == fvVowel && iSylVowel == -1) { // the first vowel in the syllable
+				iSylVowel = i;
+			}
+
+			if (inFV.getByteFeature(fiSylEnd) == 0 && inFV.getByteFeature(fiLR) == fvLR_R) { // last segment in syllable
+				writeFeatureVectors(out, iCurrent, iSylVowel, i, unitDurs);
+				iSylVowel = -1;
+				unitDurs.clear();
+				iCurrent = i + 1;
+			}
+
+		}
+
+		assert numUnits - iCurrent == unitDurs.size();
+		writeFeatureVectors(out, iCurrent, iSylVowel, numUnits - 1, unitDurs);
+	}
+
+	private void writeFeatureVectors(DataOutput out, int iFirst, int iVowel, int iLast, List<Float> unitDurs) throws IOException {
+		float[] coeffs = null;
+		if (iVowel != -1) { // Syllable contains a vowel
+			coeffs = contours.getFeatureVector(iVowel).getContinuousFeatures();
+			boolean isZero = true;
+			for (int c = 0; c < coeffs.length; c++) {
+				if (coeffs[c] != 0) {
+					isZero = false;
+					break;
+				}
+			}
+			if (isZero) {
+				coeffs = null;
+			}
+		}
+		assert unitDurs.size() == iLast - iFirst + 1;
+
+		float sylDur = 0;
+		for (int i = 0, max = unitDurs.size(); i < max; i++) {
+			sylDur += unitDurs.get(i);
+		}
+		// System.out.println("Syl dur: "+sylDur+", "+unitDurs.size()+" units");
+
+		float uStart = 0, uEnd = 0;
+		for (int i = 0; iFirst + i <= iLast; i++) {
+			float logF0 = Float.NaN;
+			float logF0delta = Float.NaN;
+			if (coeffs != null && unitDurs.get(i) > 0) {
+				float relUStart = uStart / sylDur; // in [0, 1[
+				float relUEnd = (uStart + unitDurs.get(i)) / sylDur; // in [0, 1[
+				double[] predUnitContour = Polynomial.generatePolynomialValues(ArrayUtils.copyFloat2Double(coeffs), 10,
+						relUStart, relUEnd);
+				// System.out.printf("From %.2f to %.2f:", relUStart, relUEnd);
+				// for (int k=0; k<predUnitContour.length; k++) System.out.printf(" %.2f", predUnitContour[k]);
+				// And fit a linear curve to this:
+				double[] unitCoeffs = Polynomial.fitPolynomial(predUnitContour, 1);
+				assert unitCoeffs.length == 2; // unitCoeffs[0] is the slope, unitCoeffs[1] the value at left end of interval.
+				// We need the f0 value in the middle of the unit:
+				logF0 = (float) (unitCoeffs[1] + 0.5 * unitCoeffs[0]);
+				logF0delta = (float) unitCoeffs[0];
+				// System.out.printf(" -- midF0=%.2f, slope=%.2f\n", logF0, logF0delta);
+			}
+			FeatureVector fv = feats.getFeatureVector(iFirst + i);
+			String line = fv.toString() + " " + unitDurs.get(i) + " " + logF0 + " " + logF0delta;
+			// System.out.println("fv: "+line);
+			FeatureVector outFV = outFeatureDefinition.toFeatureVector(0, line);
+			outFV.writeTo(out);
+			uStart += unitDurs.get(i);
+		}
+	}
+
+	/**
+	 * Write the header of this feature file to the given DataOutput
+	 * 
+	 * @param out
+	 * @throws IOException
+	 */
+	protected void writeHeaderTo(DataOutput out) throws IOException {
+		new MaryHeader(MaryHeader.UNITFEATS).writeTo(out);
+		outFeatureDefinition.writeBinaryTo(out);
+	}
+
+	/**
+	 * Provide the progress of computation, in percent, or -1 if that feature is not implemented.
+	 * 
+	 * @return -1 if not implemented, or an integer between 0 and 100.
+	 */
+	public int getProgress() {
+		return percent;
+	}
+
+	/**
+	 * @param args
+	 */
+	public static void main(String[] args) throws Exception {
+		AcousticFeatureFileWriter acfeatsWriter = new AcousticFeatureFileWriter();
+		DatabaseLayout db = new DatabaseLayout(acfeatsWriter);
+		acfeatsWriter.compute();
+	}
 
 }
-

marytts-builder/src/main/java/marytts/tools/voiceimport/AllophonesExtractor.java

@@ -48,206 +48,204 @@ import org.w3c.dom.Node;
  * 
  */
 public class AllophonesExtractor extends VoiceImportComponent {
-    protected File textDir;
+	protected File textDir;
 
-    protected File promptAllophonesDir;
+	protected File promptAllophonesDir;
 
-    protected String featsExt = ".xml";
+	protected String featsExt = ".xml";
 
-    protected String locale;
+	protected String locale;
 
-    protected MaryHttpClient mary;
+	protected MaryHttpClient mary;
 
-    protected String maryInputType;
+	protected String maryInputType;
 
-    protected String maryOutputType;
+	protected String maryOutputType;
 
-    protected DatabaseLayout db = null;
+	protected DatabaseLayout db = null;
 
-    protected int percent = 0;
+	protected int percent = 0;
 
-    public String STYLEDEFINITIONFILE = getName() + ".styleDefinitionFile";
+	public String STYLEDEFINITIONFILE = getName() + ".styleDefinitionFile";
 
-    // StyleDefinitionFileReader serves as switch for style processing; disabled if null:
-    protected BasenameClassificationDefinitionFileReader styleDefinition;
+	// StyleDefinitionFileReader serves as switch for style processing; disabled if null:
+	protected BasenameClassificationDefinitionFileReader styleDefinition;
 
-    public String getName() {
-        return "AllophonesExtractor";
-    }
+	public String getName() {
+		return "AllophonesExtractor";
+	}
 
-    @Override
-    protected void initialiseComp() {
-        locale = db.getProp(db.LOCALE);
-        mary = null; // initialised only if needed
-        promptAllophonesDir = new File(db.getProp(db.PROMPTALLOPHONESDIR));
-        if (!promptAllophonesDir.exists()) {
-            System.out.println("Allophones directory does not exist; ");
-            if (!promptAllophonesDir.mkdir()) {
-                throw new Error("Could not create ALLOPHONES");
-            }
-            System.out.println("Created successfully.\n");
-        }
-        maryInputType = "RAWMARYXML";
-        maryOutputType = "ALLOPHONES";
+	@Override
+	protected void initialiseComp() {
+		locale = db.getProp(db.LOCALE);
+		mary = null; // initialised only if needed
+		promptAllophonesDir = new File(db.getProp(db.PROMPTALLOPHONESDIR));
+		if (!promptAllophonesDir.exists()) {
+			System.out.println("Allophones directory does not exist; ");
+			if (!promptAllophonesDir.mkdir()) {
+				throw new Error("Could not create ALLOPHONES");
+			}
+			System.out.println("Created successfully.\n");
+		}
+		maryInputType = "RAWMARYXML";
+		maryOutputType = "ALLOPHONES";
 
-        // if styleDefinitionFileName is provided, try to initialize StyleDefinitionFileReader:
-        String styleDefinitionFileName = getProp(STYLEDEFINITIONFILE);
-        if (!styleDefinitionFileName.equals("")) {
-            try {
-                styleDefinition = new BasenameClassificationDefinitionFileReader(styleDefinitionFileName);
-                if (!styleDefinition.fileOK) {
-                    System.err.println("There were problems parsing " + styleDefinitionFileName);
-                }
-            } catch (IOException e) {
-                System.err.println("Warning: style definition file " + styleDefinitionFileName
-                        + " could not be opened, styles will not be used!");
-            }
-        }
-    }
+		// if styleDefinitionFileName is provided, try to initialize StyleDefinitionFileReader:
+		String styleDefinitionFileName = getProp(STYLEDEFINITIONFILE);
+		if (!styleDefinitionFileName.equals("")) {
+			try {
+				styleDefinition = new BasenameClassificationDefinitionFileReader(styleDefinitionFileName);
+				if (!styleDefinition.fileOK) {
+					System.err.println("There were problems parsing " + styleDefinitionFileName);
+				}
+			} catch (IOException e) {
+				System.err.println("Warning: style definition file " + styleDefinitionFileName
+						+ " could not be opened, styles will not be used!");
+			}
+		}
+	}
 
-    public SortedMap<String, String> getDefaultProps(DatabaseLayout theDb) {
-        this.db = theDb;
-        if (props == null) {
-            props = new TreeMap<String, String>();
-            props.put(STYLEDEFINITIONFILE, ""); // empty string -> disabled
-//            props.put(STYLEDEFINITIONFILE, db.getProp(db.CONFIGDIR) + "styleDefinition.txt"); // disabled by default
-        }
-        return props;
-    }
+	public SortedMap<String, String> getDefaultProps(DatabaseLayout theDb) {
+		this.db = theDb;
+		if (props == null) {
+			props = new TreeMap<String, String>();
+			props.put(STYLEDEFINITIONFILE, ""); // empty string -> disabled
+			// props.put(STYLEDEFINITIONFILE, db.getProp(db.CONFIGDIR) + "styleDefinition.txt"); // disabled by default
+		}
+		return props;
+	}
 
-    protected void setupHelp() {
-        props2Help = new TreeMap<String, String>();
-        props2Help
-                .put(
-                        STYLEDEFINITIONFILE,
-                        "Text file defining speaking style using glob patterns &ndash; to disable styles, leave this blank."
-                                + "<p>Each line in the file should have the format:"
-                                + "<pre>GLOB = style</pre>"
-                                + "where <tt>GLOB</tt> is a glob expression (e.g. <tt>ob_*</tt> to match all basenames that start with <tt>ob_</tt>)."
-                                + "<p>Lines that are empty or start with <tt>#</tt> are ignored.");
-    }
+	protected void setupHelp() {
+		props2Help = new TreeMap<String, String>();
+		props2Help
+				.put(STYLEDEFINITIONFILE,
+						"Text file defining speaking style using glob patterns &ndash; to disable styles, leave this blank."
+								+ "<p>Each line in the file should have the format:"
+								+ "<pre>GLOB = style</pre>"
+								+ "where <tt>GLOB</tt> is a glob expression (e.g. <tt>ob_*</tt> to match all basenames that start with <tt>ob_</tt>)."
+								+ "<p>Lines that are empty or start with <tt>#</tt> are ignored.");
+	}
 
-    public MaryHttpClient getMaryClient() throws IOException {
-        if (mary == null) {
-            try {
-                Address server = new Address(db.getProp(db.MARYSERVERHOST), Integer.parseInt(db.getProp(db.MARYSERVERPORT)));
-                mary = new MaryHttpClient(server);
-            } catch (IOException e) {
-                IOException myIOE = new IOException("Could not connect to Maryserver at " + db.getProp(db.MARYSERVERHOST) + " "
-                        + db.getProp(db.MARYSERVERPORT));
-                myIOE.initCause(e);
-                throw myIOE;
-            }
-        }
-        return mary;
-    }
+	public MaryHttpClient getMaryClient() throws IOException {
+		if (mary == null) {
+			try {
+				Address server = new Address(db.getProp(db.MARYSERVERHOST), Integer.parseInt(db.getProp(db.MARYSERVERPORT)));
+				mary = new MaryHttpClient(server);
+			} catch (IOException e) {
+				IOException myIOE = new IOException("Could not connect to Maryserver at " + db.getProp(db.MARYSERVERHOST) + " "
+						+ db.getProp(db.MARYSERVERPORT));
+				myIOE.initCause(e);
+				throw myIOE;
+			}
+		}
+		return mary;
+	}
 
-    public boolean compute() throws IOException, MaryConfigurationException {
-        String inputDir = db.getProp(db.TEXTDIR);
-        textDir = new File(inputDir);
-        System.out.println("Computing ALLOPHONES files for " + bnl.getLength() + " files");
-        for (int i = 0; i < bnl.getLength(); i++) {
-            percent = 100 * i / bnl.getLength();
-            generateAllophonesFile(bnl.getName(i));
-            System.out.println("    " + bnl.getName(i));
-        }
-        System.out.println("...Done.");
-        return true;
-    }
+	public boolean compute() throws IOException, MaryConfigurationException {
+		String inputDir = db.getProp(db.TEXTDIR);
+		textDir = new File(inputDir);
+		System.out.println("Computing ALLOPHONES files for " + bnl.getLength() + " files");
+		for (int i = 0; i < bnl.getLength(); i++) {
+			percent = 100 * i / bnl.getLength();
+			generateAllophonesFile(bnl.getName(i));
+			System.out.println("    " + bnl.getName(i));
+		}
+		System.out.println("...Done.");
+		return true;
+	}
 
-    /**
-     * 
-     * @param basename
-     * @throws IOException
-     */
-    public void generateAllophonesFile(String basename)
-    throws IOException, MaryConfigurationException {
-        Locale localVoice = MaryUtils.string2locale(locale);
-        String xmlLocale = MaryUtils.locale2xmllang(localVoice);
-        String inputDir = db.getProp(db.TEXTDIR);
-        String outputDir = promptAllophonesDir.getAbsolutePath();
-        String fullFileName = inputDir + File.separator + basename + db.getProp(db.TEXTEXT);
+	/**
+	 * 
+	 * @param basename
+	 * @throws IOException
+	 */
+	public void generateAllophonesFile(String basename) throws IOException, MaryConfigurationException {
+		Locale localVoice = MaryUtils.string2locale(locale);
+		String xmlLocale = MaryUtils.locale2xmllang(localVoice);
+		String inputDir = db.getProp(db.TEXTDIR);
+		String outputDir = promptAllophonesDir.getAbsolutePath();
+		String fullFileName = inputDir + File.separator + basename + db.getProp(db.TEXTEXT);
 
-        // this string controls whether style attributes are inserted into the prompt_allophones ("" -> disabled):
-        String style = "";
-        if (styleDefinition != null) {
-            style = getStyleFromStyleDefinition(basename);
-        }
+		// this string controls whether style attributes are inserted into the prompt_allophones ("" -> disabled):
+		String style = "";
+		if (styleDefinition != null) {
+			style = getStyleFromStyleDefinition(basename);
+		}
 
-        File textFile = new File(fullFileName);
-        String text = FileUtils.readFileToString(textFile, "UTF-8");
+		File textFile = new File(fullFileName);
+		String text = FileUtils.readFileToString(textFile, "UTF-8");
 
-        // First, test if there is a corresponding .rawmaryxml file in textdir:
-        File rawmaryxmlFile = new File(db.getProp(db.MARYXMLDIR) + File.separator + basename + db.getProp(db.MARYXMLEXT));
-        if (rawmaryxmlFile.exists()) {
-            if (style.isEmpty()) {
-                // just pass through the raw file:
-                text = FileUtils.readFileToString(rawmaryxmlFile, "UTF-8");
-            } else {
-                // parse the .rawmaryxml file:
-                Document document = null;
-                try {
-                    document = DomUtils.parseDocument(rawmaryxmlFile);
-                } catch (Exception e) {
-                    throw new IOException("Error parsing RAWMARYXML file: " + rawmaryxmlFile.getName(), e);
-                }
+		// First, test if there is a corresponding .rawmaryxml file in textdir:
+		File rawmaryxmlFile = new File(db.getProp(db.MARYXMLDIR) + File.separator + basename + db.getProp(db.MARYXMLEXT));
+		if (rawmaryxmlFile.exists()) {
+			if (style.isEmpty()) {
+				// just pass through the raw file:
+				text = FileUtils.readFileToString(rawmaryxmlFile, "UTF-8");
+			} else {
+				// parse the .rawmaryxml file:
+				Document document = null;
+				try {
+					document = DomUtils.parseDocument(rawmaryxmlFile);
+				} catch (Exception e) {
+					throw new IOException("Error parsing RAWMARYXML file: " + rawmaryxmlFile.getName(), e);
+				}
 
-                // get the <maryxml> node:
-                Node maryXmlNode = document.getDocumentElement();
-                Node firstMaryXmlChild = maryXmlNode.getFirstChild();
-                Node lastMaryXmlChild = maryXmlNode.getLastChild();
-                // wrap the <maryxml>'s content in new <prosody> element...
-                Element topLevelProsody = DomUtils.encloseNodesWithNewElement(firstMaryXmlChild, lastMaryXmlChild,
-                        MaryXML.PROSODY);
-                // ...and set its style attribute:
-                topLevelProsody.setAttribute("style", style);
+				// get the <maryxml> node:
+				Node maryXmlNode = document.getDocumentElement();
+				Node firstMaryXmlChild = maryXmlNode.getFirstChild();
+				Node lastMaryXmlChild = maryXmlNode.getLastChild();
+				// wrap the <maryxml>'s content in new <prosody> element...
+				Element topLevelProsody = DomUtils.encloseNodesWithNewElement(firstMaryXmlChild, lastMaryXmlChild,
+						MaryXML.PROSODY);
+				// ...and set its style attribute:
+				topLevelProsody.setAttribute("style", style);
 
-                // convert the document to the text string: 
-                text = DomUtils.document2String(document);
-            }
-        } else {
-            String prosodyOpeningTag = "";
-            String prosodyClosingTag = "";
-            if (!style.isEmpty()) {
-                prosodyOpeningTag = String.format("<%s style=\"%s\">\n", MaryXML.PROSODY, style);
-                prosodyClosingTag = String.format("</%s>\n", MaryXML.PROSODY);
-            }
-            text = getMaryXMLHeaderWithInitialBoundary(xmlLocale) + prosodyOpeningTag + text + prosodyClosingTag + "</maryxml>";
-        }
+				// convert the document to the text string:
+				text = DomUtils.document2String(document);
+			}
+		} else {
+			String prosodyOpeningTag = "";
+			String prosodyClosingTag = "";
+			if (!style.isEmpty()) {
+				prosodyOpeningTag = String.format("<%s style=\"%s\">\n", MaryXML.PROSODY, style);
+				prosodyClosingTag = String.format("</%s>\n", MaryXML.PROSODY);
+			}
+			text = getMaryXMLHeaderWithInitialBoundary(xmlLocale) + prosodyOpeningTag + text + prosodyClosingTag + "</maryxml>";
+		}
 
-        OutputStream os = new BufferedOutputStream(new FileOutputStream(new File(outputDir, basename + featsExt)));
-        MaryHttpClient maryClient = getMaryClient();
-        maryClient.process(text, maryInputType, maryOutputType, db.getProp(db.LOCALE), null, null, os);
-        os.flush();
-        os.close();
-    }
+		OutputStream os = new BufferedOutputStream(new FileOutputStream(new File(outputDir, basename + featsExt)));
+		MaryHttpClient maryClient = getMaryClient();
+		maryClient.process(text, maryInputType, maryOutputType, db.getProp(db.LOCALE), null, null, os);
+		os.flush();
+		os.close();
+	}
 
-    public static String getMaryXMLHeaderWithInitialBoundary(String locale) // wtf?
-    {
-        return "<?xml version=\"1.0\" encoding=\"UTF-8\" ?>\n" + "<maryxml version=\"0.4\"\n"
-                + "xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\"\n" + "xmlns=\"http://mary.dfki.de/2002/MaryXML\"\n"
-                + "xml:lang=\"" + locale + "\">\n" + "<boundary  breakindex=\"2\" duration=\"100\"/>\n";
+	public static String getMaryXMLHeaderWithInitialBoundary(String locale) // wtf?
+	{
+		return "<?xml version=\"1.0\" encoding=\"UTF-8\" ?>\n" + "<maryxml version=\"0.4\"\n"
+				+ "xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\"\n" + "xmlns=\"http://mary.dfki.de/2002/MaryXML\"\n"
+				+ "xml:lang=\"" + locale + "\">\n" + "<boundary  breakindex=\"2\" duration=\"100\"/>\n";
 
-    }
+	}
 
-    /**
-     * Get style for basename from style definition file. Do this by matching basename against a glob pattern.
-     * 
-     * @author steiner
-     * @param basename
-     * @return style as String
-     */
-    private String getStyleFromStyleDefinition(String basename) {
-        return styleDefinition.getValue(basename);
-    }
+	/**
+	 * Get style for basename from style definition file. Do this by matching basename against a glob pattern.
+	 * 
+	 * @author steiner
+	 * @param basename
+	 * @return style as String
+	 */
+	private String getStyleFromStyleDefinition(String basename) {
+		return styleDefinition.getValue(basename);
+	}
 
-    /**
-     * Provide the progress of computation, in percent, or -1 if that feature is not implemented.
-     * 
-     * @return -1 if not implemented, or an integer between 0 and 100.
-     */
-    public int getProgress() {
-        return percent;
-    }
+	/**
+	 * Provide the progress of computation, in percent, or -1 if that feature is not implemented.
+	 * 
+	 * @return -1 if not implemented, or an integer between 0 and 100.
+	 */
+	public int getProgress() {
+		return percent;
+	}
 
 }

marytts-builder/src/main/java/marytts/tools/voiceimport/DurationCARTTrainer.java

@@ -19,7 +19,6 @@
  */
 package marytts.tools.voiceimport;
 
-
 import java.io.BufferedReader;
 import java.io.File;
 import java.io.FileOutputStream;
@@ -42,215 +41,200 @@ import marytts.unitselection.data.TimelineReader;
 import marytts.unitselection.data.Unit;
 import marytts.unitselection.data.UnitFileReader;
 
-
 /**
- * A class which converts a text file in festvox format
- * into a one-file-per-utterance format in a given directory.
+ * A class which converts a text file in festvox format into a one-file-per-utterance format in a given directory.
+ * 
  * @author schroed
  *
  */
-public class DurationCARTTrainer extends VoiceImportComponent
-{
-    protected File unitlabelDir;
-    protected File unitfeatureDir;
-    protected File durationDir;
-    protected File durationFeatsFile;
-    protected File durationDescFile;
-    protected File wagonTreeFile;
-    protected DatabaseLayout db = null;
-    protected int percent = 0;
-    protected boolean useStepwiseTraining = false;
-    
-    private final String name = "DurationCARTTrainer";
-    public final String DURTREE = name+".durTree";
-    public final String STEPWISETRAINING = name+".stepwiseTraining";
-    public final String FEATUREFILE = name+".featureFile";
-    public final String UNITFILE = name+".unitFile";
-    public final String WAVETIMELINE = name+".waveTimeline";
-    public final String ISHNMTIMELINE = name+".isHnmTimeline";
+public class DurationCARTTrainer extends VoiceImportComponent {
+	protected File unitlabelDir;
+	protected File unitfeatureDir;
+	protected File durationDir;
+	protected File durationFeatsFile;
+	protected File durationDescFile;
+	protected File wagonTreeFile;
+	protected DatabaseLayout db = null;
+	protected int percent = 0;
+	protected boolean useStepwiseTraining = false;
 
-    public String getName(){
-        return name;
-    }
-    
-    @Override
-    protected void initialiseComp()
-    {       
-        this.unitlabelDir = new File(db.getProp(DatabaseLayout.PHONELABDIR));
-        this.unitfeatureDir = new File(db.getProp(DatabaseLayout.PHONEFEATUREDIR));
-        String durDir = db.getProp(DatabaseLayout.TEMPDIR);
-        this.durationDir = new File(durDir);
-        if (!durationDir.exists()){
-            System.out.print("temp dir "+durDir
-                    +" does not exist; ");
-            if (!durationDir.mkdir()){
-                throw new Error("Could not create DURDIR");
-            }
-            System.out.print("Created successfully.\n");
-        }         
-        this.durationFeatsFile = new File(durDir+"dur.feats");
-        this.durationDescFile = new File(durDir+"dur.desc");
-        this.wagonTreeFile = new File(durDir+"dur.tree");
-        this.useStepwiseTraining = Boolean.valueOf(getProp(STEPWISETRAINING)).booleanValue();
+	private final String name = "DurationCARTTrainer";
+	public final String DURTREE = name + ".durTree";
+	public final String STEPWISETRAINING = name + ".stepwiseTraining";
+	public final String FEATUREFILE = name + ".featureFile";
+	public final String UNITFILE = name + ".unitFile";
+	public final String WAVETIMELINE = name + ".waveTimeline";
+	public final String ISHNMTIMELINE = name + ".isHnmTimeline";
 
-    }
-    
-     public SortedMap<String, String> getDefaultProps(DatabaseLayout dbl){
-        this.db = dbl;
-        if (props == null){
-            props = new TreeMap<String, String>();
-            props.put(STEPWISETRAINING,"false");
-            props.put(FEATUREFILE, db.getProp(DatabaseLayout.FILEDIR)
-                    +"phoneFeatures"+db.getProp(DatabaseLayout.MARYEXT));
-            props.put(UNITFILE, db.getProp(DatabaseLayout.FILEDIR)
-                    +"phoneUnits"+db.getProp(DatabaseLayout.MARYEXT));
-            props.put(WAVETIMELINE, db.getProp(DatabaseLayout.FILEDIR)
-                    +"timeline_waveforms"+db.getProp(DatabaseLayout.MARYEXT)); 
-            props.put(ISHNMTIMELINE, "false"); 
-            props.put(DURTREE,db.getProp(DatabaseLayout.FILEDIR)
-                    +"dur.tree");                    
-        }
-       return props;
-    }
-     
-     protected void setupHelp(){
-         props2Help = new TreeMap<String, String>();
-         props2Help.put(STEPWISETRAINING,"\"false\" or \"true\" ???????????????????????????????????????????????????????????");
-         props2Help.put(FEATUREFILE, "file containing all phone units and their target cost features");
-         props2Help.put(UNITFILE, "file containing all phone units");
-         props2Help.put(WAVETIMELINE, "file containing all waveforms or models that can genarate them"); 
-         props2Help.put(ISHNMTIMELINE, "file containing all wave files"); 
-         props2Help.put(DURTREE,"file containing the duration CART. Will be created by this module");
-     }
+	public String getName() {
+		return name;
+	}
 
-    @Override
-    public boolean compute() throws IOException, MaryConfigurationException
-    {
-        FeatureFileReader featureFile = 
-            FeatureFileReader.getFeatureFileReader(getProp(FEATUREFILE));
-        UnitFileReader unitFile = new UnitFileReader(getProp(UNITFILE));
-        TimelineReader waveTimeline = null;
-        if (getProp(ISHNMTIMELINE).compareToIgnoreCase("true")==0)
-            waveTimeline = new HnmTimelineReader(getProp(WAVETIMELINE));
-        else
-            waveTimeline = new TimelineReader(getProp(WAVETIMELINE));
+	@Override
+	protected void initialiseComp() {
+		this.unitlabelDir = new File(db.getProp(DatabaseLayout.PHONELABDIR));
+		this.unitfeatureDir = new File(db.getProp(DatabaseLayout.PHONEFEATUREDIR));
+		String durDir = db.getProp(DatabaseLayout.TEMPDIR);
+		this.durationDir = new File(durDir);
+		if (!durationDir.exists()) {
+			System.out.print("temp dir " + durDir + " does not exist; ");
+			if (!durationDir.mkdir()) {
+				throw new Error("Could not create DURDIR");
+			}
+			System.out.print("Created successfully.\n");
+		}
+		this.durationFeatsFile = new File(durDir + "dur.feats");
+		this.durationDescFile = new File(durDir + "dur.desc");
+		this.wagonTreeFile = new File(durDir + "dur.tree");
+		this.useStepwiseTraining = Boolean.valueOf(getProp(STEPWISETRAINING)).booleanValue();
 
-        PrintWriter toFeaturesFile = new PrintWriter(new FileOutputStream(durationFeatsFile));
-        System.out.println("Duration CART trainer: exporting duration features");
+	}
 
-        FeatureDefinition featureDefinition = featureFile.getFeatureDefinition();
-        int nUnits = 0;
-        for (int i=0, len=unitFile.getNumberOfUnits(); i<len; i++) {
-            // We estimate that feature extraction takes 1/10 of the total time
-            // (that's probably wrong, but never mind)
-            percent = 10*i/len;
-            Unit u = unitFile.getUnit(i);
-            float dur = u.duration / (float) unitFile.getSampleRate();
-            if (dur >= 0.01) { // enforce a minimum duration for training data
-                toFeaturesFile.println(dur + " " + featureDefinition.toFeatureString(featureFile.getFeatureVector(i)));
-                nUnits++;
-            }
-        }
-        if (useStepwiseTraining) percent = 1;
-        else percent = 10;
-        toFeaturesFile.close();
-        System.out.println("Duration features extracted for "+nUnits+" units");
-        
-        PrintWriter toDesc = new PrintWriter(new FileOutputStream(durationDescFile));
-        generateFeatureDescriptionForWagon(featureDefinition, toDesc);
-        toDesc.close();
-        
-        boolean ok = false;
-        // Now, call wagon
-        WagonCaller wagonCaller = new WagonCaller(db.getProp(DatabaseLayout.ESTDIR), null);        
-        if (useStepwiseTraining) {
-            // Split the data set in training and test part:
-            // TODO: hardcoded path = EVIL
-            Process traintest = Runtime.getRuntime().exec("/project/mary/Festival/festvox/src/general/traintest "+durationFeatsFile.getAbsolutePath());
-             try {
-                traintest.waitFor();
-             } catch (InterruptedException ie) {}
-             ok = wagonCaller.callWagon("-data "+durationFeatsFile.getAbsolutePath()+".train"
-                     +" -test "+durationFeatsFile.getAbsolutePath()+".test -stepwise"
-                     +" -desc "+durationDescFile.getAbsolutePath()
-                     +" -stop 10 "
-                     +" -output "+wagonTreeFile.getAbsolutePath());
-        } else {
-            ok = wagonCaller.callWagon("-data "+durationFeatsFile.getAbsolutePath()
-                    +" -desc "+durationDescFile.getAbsolutePath()
-                    +" -stop 10 "
-                    +" -output "+wagonTreeFile.getAbsolutePath());            
-        }
-        if(ok){
-            String destinationFile = getProp(DURTREE);
-            WagonCARTReader wagonDURReader = new WagonCARTReader(LeafType.FloatLeafNode);
-            Node rootNode = wagonDURReader.load(new BufferedReader(new FileReader(wagonTreeFile)), featureDefinition);
-            CART durCart = new CART(rootNode, featureDefinition);
-            MaryCARTWriter wwdur = new MaryCARTWriter();
-            wwdur.dumpMaryCART(durCart, destinationFile);
-        }
-        percent = 100;
-        return ok;
+	public SortedMap<String, String> getDefaultProps(DatabaseLayout dbl) {
+		this.db = dbl;
+		if (props == null) {
+			props = new TreeMap<String, String>();
+			props.put(STEPWISETRAINING, "false");
+			props.put(FEATUREFILE, db.getProp(DatabaseLayout.FILEDIR) + "phoneFeatures" + db.getProp(DatabaseLayout.MARYEXT));
+			props.put(UNITFILE, db.getProp(DatabaseLayout.FILEDIR) + "phoneUnits" + db.getProp(DatabaseLayout.MARYEXT));
+			props.put(WAVETIMELINE,
+					db.getProp(DatabaseLayout.FILEDIR) + "timeline_waveforms" + db.getProp(DatabaseLayout.MARYEXT));
+			props.put(ISHNMTIMELINE, "false");
+			props.put(DURTREE, db.getProp(DatabaseLayout.FILEDIR) + "dur.tree");
+		}
+		return props;
+	}
 
-    }
-    
-    
-    private void generateFeatureDescriptionForWagon(FeatureDefinition fd, PrintWriter out)
-    {
-        out.println("(");
-        out.println("(segment_duration float)");
-        int nDiscreteFeatures = fd.getNumberOfByteFeatures()+fd.getNumberOfShortFeatures();
-        for (int i=0, n=fd.getNumberOfFeatures(); i<n; i++) {            
-            out.print("( ");
-            out.print(fd.getFeatureName(i));
-            if (i<nDiscreteFeatures) { // list values
-                if (fd.getNumberOfValues(i) == 20 && fd.getFeatureValueAsString(i, 19).equals("19")) {
-                    // one of our pseudo-floats
-                    out.println(" float )");
-                } else { // list the values
-                    for (int v=0, vmax=fd.getNumberOfValues(i); v<vmax; v++) {
-                        out.print("  ");
-                        String val = fd.getFeatureValueAsString(i, v);
-                        if (val.indexOf('"') != -1) {
-                            StringBuilder buf = new StringBuilder();
-                            for (int c=0; c<val.length(); c++) {
-                                char ch = val.charAt(c);
-                                if (ch == '"') buf.append("\\\"");
-                                else buf.append(ch);
-                            }
-                            val = buf.toString();
-                        }
-                        out.print("\""+val+"\"");
-                    }
-                    out.println(" )");
-                }
-            } else { // float feature
-                    out.println(" float )");
-            }
+	protected void setupHelp() {
+		props2Help = new TreeMap<String, String>();
+		props2Help.put(STEPWISETRAINING, "\"false\" or \"true\" ???????????????????????????????????????????????????????????");
+		props2Help.put(FEATUREFILE, "file containing all phone units and their target cost features");
+		props2Help.put(UNITFILE, "file containing all phone units");
+		props2Help.put(WAVETIMELINE, "file containing all waveforms or models that can genarate them");
+		props2Help.put(ISHNMTIMELINE, "file containing all wave files");
+		props2Help.put(DURTREE, "file containing the duration CART. Will be created by this module");
+	}
 
-        }
-        out.println(")");
-    }
+	@Override
+	public boolean compute() throws IOException, MaryConfigurationException {
+		FeatureFileReader featureFile = FeatureFileReader.getFeatureFileReader(getProp(FEATUREFILE));
+		UnitFileReader unitFile = new UnitFileReader(getProp(UNITFILE));
+		TimelineReader waveTimeline = null;
+		if (getProp(ISHNMTIMELINE).compareToIgnoreCase("true") == 0)
+			waveTimeline = new HnmTimelineReader(getProp(WAVETIMELINE));
+		else
+			waveTimeline = new TimelineReader(getProp(WAVETIMELINE));
 
-    
-    /**
-     * Provide the progress of computation, in percent, or -1 if
-     * that feature is not implemented.
-     * @return -1 if not implemented, or an integer between 0 and 100.
-     */
-    public int getProgress()
-    {
-        return percent;
-    }
+		PrintWriter toFeaturesFile = new PrintWriter(new FileOutputStream(durationFeatsFile));
+		System.out.println("Duration CART trainer: exporting duration features");
 
+		FeatureDefinition featureDefinition = featureFile.getFeatureDefinition();
+		int nUnits = 0;
+		for (int i = 0, len = unitFile.getNumberOfUnits(); i < len; i++) {
+			// We estimate that feature extraction takes 1/10 of the total time
+			// (that's probably wrong, but never mind)
+			percent = 10 * i / len;
+			Unit u = unitFile.getUnit(i);
+			float dur = u.duration / (float) unitFile.getSampleRate();
+			if (dur >= 0.01) { // enforce a minimum duration for training data
+				toFeaturesFile.println(dur + " " + featureDefinition.toFeatureString(featureFile.getFeatureVector(i)));
+				nUnits++;
+			}
+		}
+		if (useStepwiseTraining)
+			percent = 1;
+		else
+			percent = 10;
+		toFeaturesFile.close();
+		System.out.println("Duration features extracted for " + nUnits + " units");
 
-    public static void main(String[] args) throws Exception
-    {
-        DurationCARTTrainer dct = new DurationCARTTrainer();
-         DatabaseLayout db = new DatabaseLayout(dct);
-         dct.compute();
-    }
+		PrintWriter toDesc = new PrintWriter(new FileOutputStream(durationDescFile));
+		generateFeatureDescriptionForWagon(featureDefinition, toDesc);
+		toDesc.close();
 
+		boolean ok = false;
+		// Now, call wagon
+		WagonCaller wagonCaller = new WagonCaller(db.getProp(DatabaseLayout.ESTDIR), null);
+		if (useStepwiseTraining) {
+			// Split the data set in training and test part:
+			// TODO: hardcoded path = EVIL
+			Process traintest = Runtime.getRuntime().exec(
+					"/project/mary/Festival/festvox/src/general/traintest " + durationFeatsFile.getAbsolutePath());
+			try {
+				traintest.waitFor();
+			} catch (InterruptedException ie) {
+			}
+			ok = wagonCaller.callWagon("-data " + durationFeatsFile.getAbsolutePath() + ".train" + " -test "
+					+ durationFeatsFile.getAbsolutePath() + ".test -stepwise" + " -desc " + durationDescFile.getAbsolutePath()
+					+ " -stop 10 " + " -output " + wagonTreeFile.getAbsolutePath());
+		} else {
+			ok = wagonCaller.callWagon("-data " + durationFeatsFile.getAbsolutePath() + " -desc "
+					+ durationDescFile.getAbsolutePath() + " -stop 10 " + " -output " + wagonTreeFile.getAbsolutePath());
+		}
+		if (ok) {
+			String destinationFile = getProp(DURTREE);
+			WagonCARTReader wagonDURReader = new WagonCARTReader(LeafType.FloatLeafNode);
+			Node rootNode = wagonDURReader.load(new BufferedReader(new FileReader(wagonTreeFile)), featureDefinition);
+			CART durCart = new CART(rootNode, featureDefinition);
+			MaryCARTWriter wwdur = new MaryCARTWriter();
+			wwdur.dumpMaryCART(durCart, destinationFile);
+		}
+		percent = 100;
+		return ok;
+
+	}
+
+	private void generateFeatureDescriptionForWagon(FeatureDefinition fd, PrintWriter out) {
+		out.println("(");
+		out.println("(segment_duration float)");
+		int nDiscreteFeatures = fd.getNumberOfByteFeatures() + fd.getNumberOfShortFeatures();
+		for (int i = 0, n = fd.getNumberOfFeatures(); i < n; i++) {
+			out.print("( ");
+			out.print(fd.getFeatureName(i));
+			if (i < nDiscreteFeatures) { // list values
+				if (fd.getNumberOfValues(i) == 20 && fd.getFeatureValueAsString(i, 19).equals("19")) {
+					// one of our pseudo-floats
+					out.println(" float )");
+				} else { // list the values
+					for (int v = 0, vmax = fd.getNumberOfValues(i); v < vmax; v++) {
+						out.print("  ");
+						String val = fd.getFeatureValueAsString(i, v);
+						if (val.indexOf('"') != -1) {
+							StringBuilder buf = new StringBuilder();
+							for (int c = 0; c < val.length(); c++) {
+								char ch = val.charAt(c);
+								if (ch == '"')
+									buf.append("\\\"");
+								else
+									buf.append(ch);
+							}
+							val = buf.toString();
+						}
+						out.print("\"" + val + "\"");
+					}
+					out.println(" )");
+				}
+			} else { // float feature
+				out.println(" float )");
+			}
+
+		}
+		out.println(")");
+	}
+
+	/**
+	 * Provide the progress of computation, in percent, or -1 if that feature is not implemented.
+	 * 
+	 * @return -1 if not implemented, or an integer between 0 and 100.
+	 */
+	public int getProgress() {
+		return percent;
+	}
+
+	public static void main(String[] args) throws Exception {
+		DurationCARTTrainer dct = new DurationCARTTrainer();
+		DatabaseLayout db = new DatabaseLayout(dct);
+		dct.compute();
+	}
 
 }
-

marytts-builder/src/main/java/marytts/tools/voiceimport/DurationSoPTrainer.java

@@ -52,404 +52,371 @@ import marytts.unitselection.data.Unit;
 import marytts.unitselection.data.UnitFileReader;
 
 /***
- * Modelling duration using Sum of products (SoP) 
- * SoP is modelled using multiple linear regression
- * Selection of features is performed with Sequential Floating Forward Search(SFFS):
- *    
+ * Modelling duration using Sum of products (SoP) SoP is modelled using multiple linear regression Selection of features is
+ * performed with Sequential Floating Forward Search(SFFS):
+ * 
  * @author marcela
  */
-public class DurationSoPTrainer extends VoiceImportComponent
-{
-  //protected String features;
-  protected DatabaseLayout db = null;
-  protected int percent = 0;
-  protected boolean success = true;
-  protected boolean interceptTerm;
-  protected boolean logDuration;
-  protected int solutionSize;
-  protected File unitlabelDir;
-  protected File unitfeatureDir;
-  
-  
-  private final String name = "DurationSoPTrainer";
-  private final String LABELDIR = name+".labelDir";
-  private final String FEATUREDIR = name+".featureDir";   
-  private final String FEATUREFILE = name+".featureFile";
-  private final String UNITFILE = name+".unitFile";
-  private final String SOLUTIONSIZE = name+".solutionSize";
-  private final String INTERCEPTTERM = name+".interceptTerm";
-  private final String LOGDURATION = name+".logDuration";
-  private final String DURSOPFILE = name+".durSopFile";
- 
-  
-  public String getName(){
-    return name;
-  }
+public class DurationSoPTrainer extends VoiceImportComponent {
+	// protected String features;
+	protected DatabaseLayout db = null;
+	protected int percent = 0;
+	protected boolean success = true;
+	protected boolean interceptTerm;
+	protected boolean logDuration;
+	protected int solutionSize;
+	protected File unitlabelDir;
+	protected File unitfeatureDir;
 
-  @Override
-  protected void initialiseComp()
-  {
-    this.unitlabelDir = new File(getProp(LABELDIR));
-    this.unitfeatureDir = new File(getProp(FEATUREDIR));
-    String rootDir = db.getProp(db.ROOTDIR);
-    this.interceptTerm =  Boolean.valueOf(getProp(INTERCEPTTERM)).booleanValue(); 
-    this.logDuration =  Boolean.valueOf(getProp(LOGDURATION)).booleanValue();
-    this.solutionSize = Integer.parseInt(getProp(SOLUTIONSIZE));
-  }
+	private final String name = "DurationSoPTrainer";
+	private final String LABELDIR = name + ".labelDir";
+	private final String FEATUREDIR = name + ".featureDir";
+	private final String FEATUREFILE = name + ".featureFile";
+	private final String UNITFILE = name + ".unitFile";
+	private final String SOLUTIONSIZE = name + ".solutionSize";
+	private final String INTERCEPTTERM = name + ".interceptTerm";
+	private final String LOGDURATION = name + ".logDuration";
+	private final String DURSOPFILE = name + ".durSopFile";
 
-  public SortedMap<String, String> getDefaultProps(DatabaseLayout dbl){
-    this.db = dbl;
-    String fileDir = db.getProp(db.FILEDIR);
-    if (props == null){
-      props = new TreeMap<String, String>();
-      String fileSeparator = System.getProperty("file.separator");
-      props.put(FEATUREDIR, db.getProp(db.ROOTDIR) + "phonefeatures" + fileSeparator);
-      props.put(LABELDIR, db.getProp(db.ROOTDIR) + "phonelab" + fileSeparator);            
-      props.put(FEATUREFILE, db.getProp(db.FILEDIR) + "phoneFeatures"+db.getProp(db.MARYEXT));
-      props.put(UNITFILE, db.getProp(db.FILEDIR) + "phoneUnits"+db.getProp(db.MARYEXT));
-      props.put(INTERCEPTTERM, "true");
-      props.put(LOGDURATION, "true");
-      props.put(SOLUTIONSIZE, "10");
-      props.put(DURSOPFILE, fileDir+"dur.sop");
-      
-    }
-   return props; 
-  }
-  
-  protected void setupHelp(){
-    props2Help = new TreeMap<String, String>();
-    props2Help.put(FEATUREDIR, "directory containing the phonefeatures");
-    props2Help.put(LABELDIR, "directory containing the phone labels");                
-    props2Help.put(FEATUREFILE, "file containing all phone units and their target cost features");
-    props2Help.put(UNITFILE, "file containing all phone units");
-    props2Help.put(INTERCEPTTERM, "whether to include interceptTerm (b0) on the solution equation : b0 + b1X1 + .. bnXn");
-    props2Help.put(LOGDURATION, "whether to use log(independent variable)");
-    props2Help.put(SOLUTIONSIZE, "size of the solution, number of dependend variables");
-    props2Help.put(DURSOPFILE, "file containing the dur SoP model. Will be created by this module");
-  }
-  
-  protected void setSuccess(boolean val)
-  {
-      success = val;
-  }  
-  
-  public boolean compute() throws Exception
-  {
- 
-    String durDir = db.getProp(db.TEMPDIR);
-    
-    String vowelsFile = durDir + "vowels.feats";
-    String consonantsFile = durDir + "consonants.feats";
-    String pauseFile = durDir + "pause.feats";
-    
-    String[] lingFactorsVowel;
-    String[] lingFactorsConsonant;
-    String[] lingFactorsPause;
-      
-    AllophoneSet allophoneSet = db.getAllophoneSet();
-    
-    FeatureFileReader featureFile = FeatureFileReader.getFeatureFileReader(getProp(FEATUREFILE));
-    UnitFileReader unitFile = new UnitFileReader(getProp(UNITFILE));
+	public String getName() {
+		return name;
+	}
 
-    FeatureDefinition featureDefinition = featureFile.getFeatureDefinition();
-    FeatureVector fv;
-    int nUnitsVowel = 0;
-    int nUnitsConsonant = 0;
-    int nUnitsPause = 0;
-    
-    //System.out.println("Feature names: " + featureDefinition.getFeatureNames());
-    // select features that will be used as linguistic factors on the regression
-    lingFactorsVowel = selectLinguisticFactors(featureDefinition.getFeatureNames(), "Select linguistic factors for vowels:");
-    lingFactorsConsonant = selectLinguisticFactors(featureDefinition.getFeatureNames(), "Select linguistic factors for consonants:");
-    lingFactorsPause = selectLinguisticFactors(featureDefinition.getFeatureNames(), "Select linguistic factors for pause:");
-    
-    // the following files contain all the feature files in columns
-    PrintWriter toVowelsFile = new PrintWriter(new FileOutputStream(vowelsFile));
-    PrintWriter toConsonantsFile = new PrintWriter(new FileOutputStream(consonantsFile));
-    PrintWriter toPauseFile = new PrintWriter(new FileOutputStream(pauseFile));
- 
-    int k = 0;
-    int numVowels=0;
-    int numConsonants=0;
-    int numPause=0;
-    // index of phone
-    int phoneIndex = featureDefinition.getFeatureIndex("phone");
-    for (int i=0, len=unitFile.getNumberOfUnits(); i<len; i++) {
-      // We estimate that feature extraction takes 1/10 of the total time
-      // (that's probably wrong, but never mind)
-      percent = 10*i/len;
-      
-      Unit u = unitFile.getUnit(i);
-      double dur = u.duration / (float) unitFile.getSampleRate();    
-       
-      fv = featureFile.getFeatureVector(i); 
-           
-      // first select pause, then vowell and last consonant phones
-      if(fv.getByteFeature(phoneIndex) > 0 && dur >= 0.01 ){  
-        if (allophoneSet.getAllophone(fv.getFeatureAsString(phoneIndex, featureDefinition)).isPause()){          
-          for(int j=0; j < lingFactorsPause.length; j++)
-            toPauseFile.print(fv.getByteFeature(featureDefinition.getFeatureIndex(lingFactorsPause[j])) + " ");            
-          if(logDuration)
-            toPauseFile.println(Math.log(dur)); // last column is the dependent variable, in this case duration
-          else
-            toPauseFile.println(dur);
-          numPause++;
-        } else if (allophoneSet.getAllophone(fv.getFeatureAsString(phoneIndex, featureDefinition)).isVowel()){          
-          for(int j=0; j < lingFactorsVowel.length; j++){
-            byte feaVal = fv.getByteFeature(featureDefinition.getFeatureIndex(lingFactorsVowel[j]));
-            toVowelsFile.print(feaVal + " ");
-          }
-          if(logDuration)
-            toVowelsFile.println(Math.log(dur)); // last column is the dependent variable, in this case duration
-          else
-            toVowelsFile.println(dur);
-          numVowels++;          
-        } else {  // everything else will be considered consonant! is this correct?
-          for(int j=0; j < lingFactorsConsonant.length; j++){
-            byte feaVal = fv.getByteFeature(featureDefinition.getFeatureIndex(lingFactorsConsonant[j]));
-            toConsonantsFile.print( feaVal + " ");
-          }
-          if(logDuration)
-            toConsonantsFile.println(Math.log(dur));
-          else
-            toConsonantsFile.println(dur);
-          numConsonants++;
-        }
-      }          
-   }
-   toVowelsFile.close();
-   toConsonantsFile.close();
-   toPauseFile.close();
-   percent = 10; 
-   int cols, rows;
+	@Override
+	protected void initialiseComp() {
+		this.unitlabelDir = new File(getProp(LABELDIR));
+		this.unitfeatureDir = new File(getProp(FEATUREDIR));
+		String rootDir = db.getProp(db.ROOTDIR);
+		this.interceptTerm = Boolean.valueOf(getProp(INTERCEPTTERM)).booleanValue();
+		this.logDuration = Boolean.valueOf(getProp(LOGDURATION)).booleanValue();
+		this.solutionSize = Integer.parseInt(getProp(SOLUTIONSIZE));
+	}
 
-   double percentToTrain = 0.7;
-   
-   // the final regression will be saved in this file, one line for vowels, one for consonants and another for pause
-   PrintWriter toSopFile = new PrintWriter(new FileOutputStream(getProp(DURSOPFILE)));
-   
-   // Save first the features definition on the output file
-   featureDefinition.writeTo(toSopFile, false);
-   toSopFile.println();
-   
-   SFFS sffs = new SFFS(solutionSize, interceptTerm, logDuration);
-  
-   System.out.println("\n==================================\nProcessing Vowels:");
-   SoP sopVowel = new SoP(featureDefinition);
-   sffs.trainModel(lingFactorsVowel, vowelsFile, numVowels, percentToTrain, sopVowel);
-   toSopFile.println("vowel");
-   sopVowel.saveSelectedFeatures(toSopFile);
-  
-   System.out.println("\n==================================\nProcessing Consonants:");
-   SoP sopConsonant = new SoP(featureDefinition);
-   sffs.trainModel(lingFactorsConsonant, consonantsFile, numConsonants, percentToTrain, sopConsonant);
-   toSopFile.println("consonant");
-   sopConsonant.saveSelectedFeatures(toSopFile);
-   
-   System.out.println("\n==================================\nProcessing Pause:");
-   SoP sopPause = new SoP(featureDefinition);
-   sffs.trainModel(lingFactorsPause, pauseFile, numPause, percentToTrain, sopPause);
-   toSopFile.println("pause");
-   sopPause.saveSelectedFeatures(toSopFile);
-   
-   toSopFile.close();
-   
-   percent = 100;
-   
-   return true;
-  }
-  
-    
-  public String[] selectLinguisticFactors(String featureNames, String label) throws IOException
-  {
-      String[] lingFactors=null;
-      String features = checkFeatureList(featureNames);
- 
-      final JFrame frame = new JFrame(label);
-      GridBagLayout gridBagLayout = new GridBagLayout();
-      GridBagConstraints gridC = new GridBagConstraints();
-      frame.getContentPane().setLayout( gridBagLayout );
-      
-      final JEditorPane editPane = new JEditorPane();
-      editPane.setPreferredSize(new Dimension(500, 500));
-      editPane.setText(features);        
-      
-      JButton saveButton = new JButton("Save");
-      saveButton.addActionListener(new ActionListener() {
-         public void actionPerformed(ActionEvent e) {                
-             setSuccess(true);
-             frame.setVisible(false);
-          }
-      });
-      JButton cancelButton = new JButton("Cancel");
-      cancelButton.addActionListener(new ActionListener() {
-          public void actionPerformed(ActionEvent e) {
-              setSuccess(false);
-              frame.setVisible(false);
-          }
-      });
-      
-      gridC.gridx = 0;
-      gridC.gridy = 0;
-      // resize scroll pane:
-      gridC.weightx = 1;
-      gridC.weighty = 1;
-      gridC.fill = GridBagConstraints.HORIZONTAL;
-      JScrollPane scrollPane = new JScrollPane(editPane);
-      scrollPane.setPreferredSize(editPane.getPreferredSize());
-      gridBagLayout.setConstraints( scrollPane, gridC );
-      frame.getContentPane().add(scrollPane);
-      gridC.gridy = 1;
-      // do not resize buttons:
-      gridC.weightx = 0;
-      gridC.weighty = 0;
-      JPanel buttonPanel = new JPanel();
-      buttonPanel.setLayout(new FlowLayout());
-      buttonPanel.add(saveButton);
-      buttonPanel.add(cancelButton);
-      gridBagLayout.setConstraints( buttonPanel, gridC );
-      frame.getContentPane().add(buttonPanel);
-      frame.pack();
-      frame.setVisible(true);
-      
-      do {
-          try {
-              Thread.sleep(10); 
-          } catch (InterruptedException e) {}
-      } while (frame.isVisible());
-      frame.dispose();
+	public SortedMap<String, String> getDefaultProps(DatabaseLayout dbl) {
+		this.db = dbl;
+		String fileDir = db.getProp(db.FILEDIR);
+		if (props == null) {
+			props = new TreeMap<String, String>();
+			String fileSeparator = System.getProperty("file.separator");
+			props.put(FEATUREDIR, db.getProp(db.ROOTDIR) + "phonefeatures" + fileSeparator);
+			props.put(LABELDIR, db.getProp(db.ROOTDIR) + "phonelab" + fileSeparator);
+			props.put(FEATUREFILE, db.getProp(db.FILEDIR) + "phoneFeatures" + db.getProp(db.MARYEXT));
+			props.put(UNITFILE, db.getProp(db.FILEDIR) + "phoneUnits" + db.getProp(db.MARYEXT));
+			props.put(INTERCEPTTERM, "true");
+			props.put(LOGDURATION, "true");
+			props.put(SOLUTIONSIZE, "10");
+			props.put(DURSOPFILE, fileDir + "dur.sop");
 
-      if (success) {
-          try{
-              lingFactors = saveFeatures(editPane.getText());                
-          } catch (Exception ex){
-              ex.printStackTrace();
-              throw new Error("Error defining replacements");
-          }
-      }
-     //return true;
-      return lingFactors;
-  }
+		}
+		return props;
+	}
 
-  private String checkFeatureList(String featureNames) throws IOException
-  {
-      String featureList = "";
-      String recommendedFeatureList = "";
-      String feaList[] = featureNames.split(" ");
-      String line;
-     
-      for(int i=0; i<feaList.length; i++){
-          line = feaList[i];
-          
-          /*
-          // CHECK: Maybe we need to exclude some features from the selection list???
-          // The following have variance 0
-          if( !(line.contains("style") ||
-                line.contains("sentence_punc") ||
-                line.contains("next_punctuation") ||
-                line.contains("prev_punctuation") ||
-                line.contains("ph_cplace") ||
-                line.contains("ph_cvox") ||
-                line.contains("ph_vc") ||
-                line.contains("onsetcoda") ||
-                line.contains("edge") )) {
-            
-               // CHECK: here i am including arbitrarily some....
-              // put in front the recomended ones: "ph_vfront","ph_vrnd","position_type","pos_in_syl"                
-              if( line.contentEquals("ph_vfront") ||
-                  line.contentEquals("ph_height") ||
-                  line.contentEquals("ph_vlng") ||
-                  line.contentEquals("ph_vrnd") ||
-                  line.contentEquals("ph_cplace") ||
-                  line.contentEquals("ph_ctype") ||
-                  line.contentEquals("ph_cvox") ||
-                  line.contentEquals("phone") ||
-                  line.contentEquals("position_type") )
-                    recommendedFeatureList += line + "\n";
-              else              
-                 featureList += line + "\n";
-          }
-          */
-          featureList += line + "\n";    
-        }
-      //return recommendedFeatureList + "\n" + featureList;
-      return featureList;
-      //return "";
+	protected void setupHelp() {
+		props2Help = new TreeMap<String, String>();
+		props2Help.put(FEATUREDIR, "directory containing the phonefeatures");
+		props2Help.put(LABELDIR, "directory containing the phone labels");
+		props2Help.put(FEATUREFILE, "file containing all phone units and their target cost features");
+		props2Help.put(UNITFILE, "file containing all phone units");
+		props2Help.put(INTERCEPTTERM, "whether to include interceptTerm (b0) on the solution equation : b0 + b1X1 + .. bnXn");
+		props2Help.put(LOGDURATION, "whether to use log(independent variable)");
+		props2Help.put(SOLUTIONSIZE, "size of the solution, number of dependend variables");
+		props2Help.put(DURSOPFILE, "file containing the dur SoP model. Will be created by this module");
+	}
 
-  }
- 
-  
-  private String[] saveFeatures(String newFeatures)
-  {     
-     String fea[] = newFeatures.split("\n");
-     String[] lingFactors = new String[fea.length];
-     System.out.print("Selected linguistic factors (" + fea.length + "):");
-     for(int i=0; i<fea.length; i++){
-       System.out.print(fea[i] + " ");
-       lingFactors[i] = fea[i];
-     }     
-     System.out.println();
-     return lingFactors;
-  }  
-  
-  
-  public int getProgress()
-  {
-      return percent;
-  }
+	protected void setSuccess(boolean val) {
+		success = val;
+	}
 
+	public boolean compute() throws Exception {
 
-  
-  public static void main(String[] args) throws Exception
-  {
-      /*DurationSoPTrainer sop = new DurationSoPTrainer(); 
-      DatabaseLayout db = new DatabaseLayout(sop);
-      sop.compute();
-      */
-      String sopFileName = "/project/mary/marcela/UnitSel-voices/slt-arctic/temp/dur.sop";
-      //String contextFile = "/project/mary/marcela/UnitSel-voices/slt-arctic/phonefeatures/arctic_a0001.pfeats";
-      File sopFile = new File(sopFileName);
-         
-      // Read dur.sop file 
-      // the first line corresponds to vowels and the second to consonants
-      String nextLine;
-      String strContext="";
-      Scanner s = null;
-      try {
-        s = new Scanner(new BufferedReader(new FileReader(sopFileName)));
-        
-        while (s.hasNext()) {
-          nextLine = s.nextLine(); 
-          if (nextLine.trim().equals("")) break;
-          else
-            strContext += nextLine + "\n";
-        }
-        // the featureDefinition is the same for vowel, consonant and Pause
-        FeatureDefinition voiceFeatDef = new FeatureDefinition(new BufferedReader(new StringReader(strContext)), false);
-        
-        // vowel line
-        if (s.hasNext()){
-          nextLine = s.nextLine();
-          System.out.println("line vowel = " + nextLine);
-          SoP sopVowel = new SoP(nextLine, voiceFeatDef);
-          sopVowel.printCoefficients();
-        }
-        
-        // consonant line
-        if (s.hasNext()){
-          nextLine = s.nextLine();
-          System.out.println("line consonants = " + nextLine);
-          SoP sopConsonants = new SoP(nextLine, voiceFeatDef);
-          sopConsonants.printCoefficients();
-        }
-                   
-      } finally {
-          if (s != null)
-            s.close();
-      }   
-      
-  }
+		String durDir = db.getProp(db.TEMPDIR);
+
+		String vowelsFile = durDir + "vowels.feats";
+		String consonantsFile = durDir + "consonants.feats";
+		String pauseFile = durDir + "pause.feats";
+
+		String[] lingFactorsVowel;
+		String[] lingFactorsConsonant;
+		String[] lingFactorsPause;
+
+		AllophoneSet allophoneSet = db.getAllophoneSet();
+
+		FeatureFileReader featureFile = FeatureFileReader.getFeatureFileReader(getProp(FEATUREFILE));
+		UnitFileReader unitFile = new UnitFileReader(getProp(UNITFILE));
+
+		FeatureDefinition featureDefinition = featureFile.getFeatureDefinition();
+		FeatureVector fv;
+		int nUnitsVowel = 0;
+		int nUnitsConsonant = 0;
+		int nUnitsPause = 0;
+
+		// System.out.println("Feature names: " + featureDefinition.getFeatureNames());
+		// select features that will be used as linguistic factors on the regression
+		lingFactorsVowel = selectLinguisticFactors(featureDefinition.getFeatureNames(), "Select linguistic factors for vowels:");
+		lingFactorsConsonant = selectLinguisticFactors(featureDefinition.getFeatureNames(),
+				"Select linguistic factors for consonants:");
+		lingFactorsPause = selectLinguisticFactors(featureDefinition.getFeatureNames(), "Select linguistic factors for pause:");
+
+		// the following files contain all the feature files in columns
+		PrintWriter toVowelsFile = new PrintWriter(new FileOutputStream(vowelsFile));
+		PrintWriter toConsonantsFile = new PrintWriter(new FileOutputStream(consonantsFile));
+		PrintWriter toPauseFile = new PrintWriter(new FileOutputStream(pauseFile));
+
+		int k = 0;
+		int numVowels = 0;
+		int numConsonants = 0;
+		int numPause = 0;
+		// index of phone
+		int phoneIndex = featureDefinition.getFeatureIndex("phone");
+		for (int i = 0, len = unitFile.getNumberOfUnits(); i < len; i++) {
+			// We estimate that feature extraction takes 1/10 of the total time
+			// (that's probably wrong, but never mind)
+			percent = 10 * i / len;
+
+			Unit u = unitFile.getUnit(i);
+			double dur = u.duration / (float) unitFile.getSampleRate();
+
+			fv = featureFile.getFeatureVector(i);
+
+			// first select pause, then vowell and last consonant phones
+			if (fv.getByteFeature(phoneIndex) > 0 && dur >= 0.01) {
+				if (allophoneSet.getAllophone(fv.getFeatureAsString(phoneIndex, featureDefinition)).isPause()) {
+					for (int j = 0; j < lingFactorsPause.length; j++)
+						toPauseFile.print(fv.getByteFeature(featureDefinition.getFeatureIndex(lingFactorsPause[j])) + " ");
+					if (logDuration)
+						toPauseFile.println(Math.log(dur)); // last column is the dependent variable, in this case duration
+					else
+						toPauseFile.println(dur);
+					numPause++;
+				} else if (allophoneSet.getAllophone(fv.getFeatureAsString(phoneIndex, featureDefinition)).isVowel()) {
+					for (int j = 0; j < lingFactorsVowel.length; j++) {
+						byte feaVal = fv.getByteFeature(featureDefinition.getFeatureIndex(lingFactorsVowel[j]));
+						toVowelsFile.print(feaVal + " ");
+					}
+					if (logDuration)
+						toVowelsFile.println(Math.log(dur)); // last column is the dependent variable, in this case duration
+					else
+						toVowelsFile.println(dur);
+					numVowels++;
+				} else { // everything else will be considered consonant! is this correct?
+					for (int j = 0; j < lingFactorsConsonant.length; j++) {
+						byte feaVal = fv.getByteFeature(featureDefinition.getFeatureIndex(lingFactorsConsonant[j]));
+						toConsonantsFile.print(feaVal + " ");
+					}
+					if (logDuration)
+						toConsonantsFile.println(Math.log(dur));
+					else
+						toConsonantsFile.println(dur);
+					numConsonants++;
+				}
+			}
+		}
+		toVowelsFile.close();
+		toConsonantsFile.close();
+		toPauseFile.close();
+		percent = 10;
+		int cols, rows;
+
+		double percentToTrain = 0.7;
+
+		// the final regression will be saved in this file, one line for vowels, one for consonants and another for pause
+		PrintWriter toSopFile = new PrintWriter(new FileOutputStream(getProp(DURSOPFILE)));
+
+		// Save first the features definition on the output file
+		featureDefinition.writeTo(toSopFile, false);
+		toSopFile.println();
+
+		SFFS sffs = new SFFS(solutionSize, interceptTerm, logDuration);
+
+		System.out.println("\n==================================\nProcessing Vowels:");
+		SoP sopVowel = new SoP(featureDefinition);
+		sffs.trainModel(lingFactorsVowel, vowelsFile, numVowels, percentToTrain, sopVowel);
+		toSopFile.println("vowel");
+		sopVowel.saveSelectedFeatures(toSopFile);
+
+		System.out.println("\n==================================\nProcessing Consonants:");
+		SoP sopConsonant = new SoP(featureDefinition);
+		sffs.trainModel(lingFactorsConsonant, consonantsFile, numConsonants, percentToTrain, sopConsonant);
+		toSopFile.println("consonant");
+		sopConsonant.saveSelectedFeatures(toSopFile);
+
+		System.out.println("\n==================================\nProcessing Pause:");
+		SoP sopPause = new SoP(featureDefinition);
+		sffs.trainModel(lingFactorsPause, pauseFile, numPause, percentToTrain, sopPause);
+		toSopFile.println("pause");
+		sopPause.saveSelectedFeatures(toSopFile);
+
+		toSopFile.close();
+
+		percent = 100;
+
+		return true;
+	}
+
+	public String[] selectLinguisticFactors(String featureNames, String label) throws IOException {
+		String[] lingFactors = null;
+		String features = checkFeatureList(featureNames);
+
+		final JFrame frame = new JFrame(label);
+		GridBagLayout gridBagLayout = new GridBagLayout();
+		GridBagConstraints gridC = new GridBagConstraints();
+		frame.getContentPane().setLayout(gridBagLayout);
+
+		final JEditorPane editPane = new JEditorPane();
+		editPane.setPreferredSize(new Dimension(500, 500));
+		editPane.setText(features);
+
+		JButton saveButton = new JButton("Save");
+		saveButton.addActionListener(new ActionListener() {
+			public void actionPerformed(ActionEvent e) {
+				setSuccess(true);
+				frame.setVisible(false);
+			}
+		});
+		JButton cancelButton = new JButton("Cancel");
+		cancelButton.addActionListener(new ActionListener() {
+			public void actionPerformed(ActionEvent e) {
+				setSuccess(false);
+				frame.setVisible(false);
+			}
+		});
+
+		gridC.gridx = 0;
+		gridC.gridy = 0;
+		// resize scroll pane:
+		gridC.weightx = 1;
+		gridC.weighty = 1;
+		gridC.fill = GridBagConstraints.HORIZONTAL;
+		JScrollPane scrollPane = new JScrollPane(editPane);
+		scrollPane.setPreferredSize(editPane.getPreferredSize());
+		gridBagLayout.setConstraints(scrollPane, gridC);
+		frame.getContentPane().add(scrollPane);
+		gridC.gridy = 1;
+		// do not resize buttons:
+		gridC.weightx = 0;
+		gridC.weighty = 0;
+		JPanel buttonPanel = new JPanel();
+		buttonPanel.setLayout(new FlowLayout());
+		buttonPanel.add(saveButton);
+		buttonPanel.add(cancelButton);
+		gridBagLayout.setConstraints(buttonPanel, gridC);
+		frame.getContentPane().add(buttonPanel);
+		frame.pack();
+		frame.setVisible(true);
+
+		do {
+			try {
+				Thread.sleep(10);
+			} catch (InterruptedException e) {
+			}
+		} while (frame.isVisible());
+		frame.dispose();
+
+		if (success) {
+			try {
+				lingFactors = saveFeatures(editPane.getText());
+			} catch (Exception ex) {
+				ex.printStackTrace();
+				throw new Error("Error defining replacements");
+			}
+		}
+		// return true;
+		return lingFactors;
+	}
+
+	private String checkFeatureList(String featureNames) throws IOException {
+		String featureList = "";
+		String recommendedFeatureList = "";
+		String feaList[] = featureNames.split(" ");
+		String line;
+
+		for (int i = 0; i < feaList.length; i++) {
+			line = feaList[i];
+
+			/*
+			 * // CHECK: Maybe we need to exclude some features from the selection list??? // The following have variance 0 if(
+			 * !(line.contains("style") || line.contains("sentence_punc") || line.contains("next_punctuation") ||
+			 * line.contains("prev_punctuation") || line.contains("ph_cplace") || line.contains("ph_cvox") ||
+			 * line.contains("ph_vc") || line.contains("onsetcoda") || line.contains("edge") )) {
+			 * 
+			 * // CHECK: here i am including arbitrarily some.... // put in front the recomended ones:
+			 * "ph_vfront","ph_vrnd","position_type","pos_in_syl" if( line.contentEquals("ph_vfront") ||
+			 * line.contentEquals("ph_height") || line.contentEquals("ph_vlng") || line.contentEquals("ph_vrnd") ||
+			 * line.contentEquals("ph_cplace") || line.contentEquals("ph_ctype") || line.contentEquals("ph_cvox") ||
+			 * line.contentEquals("phone") || line.contentEquals("position_type") ) recommendedFeatureList += line + "\n"; else
+			 * featureList += line + "\n"; }
+			 */
+			featureList += line + "\n";
+		}
+		// return recommendedFeatureList + "\n" + featureList;
+		return featureList;
+		// return "";
+
+	}
+
+	private String[] saveFeatures(String newFeatures) {
+		String fea[] = newFeatures.split("\n");
+		String[] lingFactors = new String[fea.length];
+		System.out.print("Selected linguistic factors (" + fea.length + "):");
+		for (int i = 0; i < fea.length; i++) {
+			System.out.print(fea[i] + " ");
+			lingFactors[i] = fea[i];
+		}
+		System.out.println();
+		return lingFactors;
+	}
+
+	public int getProgress() {
+		return percent;
+	}
+
+	public static void main(String[] args) throws Exception {
+		/*
+		 * DurationSoPTrainer sop = new DurationSoPTrainer(); DatabaseLayout db = new DatabaseLayout(sop); sop.compute();
+		 */
+		String sopFileName = "/project/mary/marcela/UnitSel-voices/slt-arctic/temp/dur.sop";
+		// String contextFile = "/project/mary/marcela/UnitSel-voices/slt-arctic/phonefeatures/arctic_a0001.pfeats";
+		File sopFile = new File(sopFileName);
+
+		// Read dur.sop file
+		// the first line corresponds to vowels and the second to consonants
+		String nextLine;
+		String strContext = "";
+		Scanner s = null;
+		try {
+			s = new Scanner(new BufferedReader(new FileReader(sopFileName)));
+
+			while (s.hasNext()) {
+				nextLine = s.nextLine();
+				if (nextLine.trim().equals(""))
+					break;
+				else
+					strContext += nextLine + "\n";
+			}
+			// the featureDefinition is the same for vowel, consonant and Pause
+			FeatureDefinition voiceFeatDef = new FeatureDefinition(new BufferedReader(new StringReader(strContext)), false);
+
+			// vowel line
+			if (s.hasNext()) {
+				nextLine = s.nextLine();
+				System.out.println("line vowel = " + nextLine);
+				SoP sopVowel = new SoP(nextLine, voiceFeatDef);
+				sopVowel.printCoefficients();
+			}
+
+			// consonant line
+			if (s.hasNext()) {
+				nextLine = s.nextLine();
+				System.out.println("line consonants = " + nextLine);
+				SoP sopConsonants = new SoP(nextLine, voiceFeatDef);
+				sopConsonants.printCoefficients();
+			}
+
+		} finally {
+			if (s != null)
+				s.close();
+		}
+
+	}
 
-  
-  
 }

marytts-builder/src/main/java/marytts/tools/voiceimport/DurationTreeTrainer.java

@@ -19,7 +19,6 @@
  */
 package marytts.tools.voiceimport;
 
-
 import java.io.IOException;
 import java.util.SortedMap;
 import java.util.TreeMap;
@@ -40,135 +39,120 @@ import marytts.unitselection.data.FeatureFileReader;
 import marytts.unitselection.data.UnitFileReader;
 import marytts.util.math.MathUtils;
 
-
 /**
- * A class which converts a text file in festvox format
- * into a one-file-per-utterance format in a given directory.
+ * A class which converts a text file in festvox format into a one-file-per-utterance format in a given directory.
+ * 
  * @author schroed
  *
  */
-public class DurationTreeTrainer extends VoiceImportComponent
-{
-    protected DatabaseLayout db = null;
-    
-    private final String name = "DurationTreeTrainer";
-    public final String DURTREE = name+".durTree";
-    public final String FEATUREFILE = name+".featureFile";
-    public final String UNITFILE = name+".unitFile";
-    public final String MAXDATA = name+".maxData";
-    public final String PROPORTIONTESTDATA = name+".propTestData";
-    public String getName(){
-        return name;
-    }
-    
- 
-     public SortedMap<String, String> getDefaultProps(DatabaseLayout theDb){
-        this.db = theDb;
-        if (props == null){
-            props = new TreeMap<String, String>();
-            String fileSeparator = System.getProperty("file.separator");
-            props.put(FEATUREFILE, db.getProp(db.FILEDIR)
-                    +"phoneFeatures"+db.getProp(db.MARYEXT));
-            props.put(UNITFILE, db.getProp(db.FILEDIR)
-                    +"phoneUnits"+db.getProp(db.MARYEXT));
-            props.put(DURTREE,db.getProp(db.FILEDIR)
-                    +"dur.graph.mry");
-            props.put(MAXDATA, "0");
-            props.put(PROPORTIONTESTDATA, "0.1");
-        }
-       return props;
-    }
-     
-     protected void setupHelp(){
-         props2Help = new TreeMap<String, String>();
-         props2Help.put(FEATUREFILE, "file containing all phone units and their target cost features");
-         props2Help.put(UNITFILE, "file containing all phone units");
-         props2Help.put(DURTREE,"file containing the duration tree. Will be created by this module");
-         props2Help.put(MAXDATA, "if >0, gives the maximum number of syllables to use for training the tree");
-         props2Help.put(PROPORTIONTESTDATA, "the proportion of the data to use as test data (choose so that 1/value is an integer)");
-     }
+public class DurationTreeTrainer extends VoiceImportComponent {
+	protected DatabaseLayout db = null;
 
-     @Override
-     public boolean compute() throws IOException, MaryConfigurationException
-     {
-        logger.info("Duration tree trainer started.");
-        FeatureFileReader featureFile = 
-            FeatureFileReader.getFeatureFileReader(getProp(FEATUREFILE));
-        UnitFileReader unitFile = new UnitFileReader(getProp(UNITFILE));
+	private final String name = "DurationTreeTrainer";
+	public final String DURTREE = name + ".durTree";
+	public final String FEATUREFILE = name + ".featureFile";
+	public final String UNITFILE = name + ".unitFile";
+	public final String MAXDATA = name + ".maxData";
+	public final String PROPORTIONTESTDATA = name + ".propTestData";
 
-        FeatureVector[] allFeatureVectors = featureFile.getFeatureVectors();
-        int maxData = Integer.parseInt(getProp(MAXDATA));
-        if (maxData == 0) maxData = allFeatureVectors.length;
-        FeatureVector[] featureVectors = new FeatureVector[Math.min(maxData, allFeatureVectors.length)];
-        System.arraycopy(allFeatureVectors, 0, featureVectors, 0, featureVectors.length);
-        logger.debug("Total of "+allFeatureVectors.length+" feature vectors -- will use "+featureVectors.length);
-        
-        AgglomerativeClusterer clusterer = new AgglomerativeClusterer(featureVectors,
-                featureFile.getFeatureDefinition(),
-                null,
-                new DurationDistanceMeasure(unitFile),
-                Float.parseFloat(getProp(PROPORTIONTESTDATA)));
-        DirectedGraphWriter writer = new DirectedGraphWriter();
-        DirectedGraph graph;
-        int iteration = 0;
-        do {
-            graph = clusterer.cluster();
-            iteration++;
-            if (graph != null) {
-                writer.saveGraph(graph, getProp(DURTREE)+".level"+iteration);
-            }            
-        } while (clusterer.canClusterMore());
+	public String getName() {
+		return name;
+	}
 
-        if (graph == null) {
-            return false;
-        }
+	public SortedMap<String, String> getDefaultProps(DatabaseLayout theDb) {
+		this.db = theDb;
+		if (props == null) {
+			props = new TreeMap<String, String>();
+			String fileSeparator = System.getProperty("file.separator");
+			props.put(FEATUREFILE, db.getProp(db.FILEDIR) + "phoneFeatures" + db.getProp(db.MARYEXT));
+			props.put(UNITFILE, db.getProp(db.FILEDIR) + "phoneUnits" + db.getProp(db.MARYEXT));
+			props.put(DURTREE, db.getProp(db.FILEDIR) + "dur.graph.mry");
+			props.put(MAXDATA, "0");
+			props.put(PROPORTIONTESTDATA, "0.1");
+		}
+		return props;
+	}
 
-        // Now replace each leaf with a FloatLeafNode containing mean and stddev
-        for (LeafNode leaf : graph.getLeafNodes()) {
-            FeatureVectorLeafNode fvLeaf = (FeatureVectorLeafNode) leaf;
-            FeatureVector[] fvs = fvLeaf.getFeatureVectors();
-            double[] dur = new double[fvs.length];
-            for (int i=0; i<fvs.length; i++) {
-                dur[i] = unitFile.getUnit(fvs[i].getUnitIndex()).duration / (float)unitFile.getSampleRate();
-            }
-            double mean = MathUtils.mean(dur);
-            double stddev = MathUtils.standardDeviation(dur, mean);
-            FloatLeafNode floatLeaf = new FloatLeafNode(new float[] {(float)stddev, (float)mean});
-            Node mother = fvLeaf.getMother();
-            assert mother != null;
-            if (mother.isDecisionNode()) {
-                ((DecisionNode)mother).replaceDaughter(floatLeaf, fvLeaf.getNodeIndex());
-            } else {
-                assert mother.isDirectedGraphNode();
-                assert ((DirectedGraphNode)mother).getLeafNode() == fvLeaf;
-                ((DirectedGraphNode)mother).setLeafNode(floatLeaf);
-            }
-        }
-        writer.saveGraph(graph, getProp(DURTREE));
-        return true;
-        
+	protected void setupHelp() {
+		props2Help = new TreeMap<String, String>();
+		props2Help.put(FEATUREFILE, "file containing all phone units and their target cost features");
+		props2Help.put(UNITFILE, "file containing all phone units");
+		props2Help.put(DURTREE, "file containing the duration tree. Will be created by this module");
+		props2Help.put(MAXDATA, "if >0, gives the maximum number of syllables to use for training the tree");
+		props2Help.put(PROPORTIONTESTDATA,
+				"the proportion of the data to use as test data (choose so that 1/value is an integer)");
+	}
 
-    }
-    
-    
-    /**
-     * Provide the progress of computation, in percent, or -1 if
-     * that feature is not implemented.
-     * @return -1 if not implemented, or an integer between 0 and 100.
-     */
-    public int getProgress()
-    {
-        return -1;
-    }
+	@Override
+	public boolean compute() throws IOException, MaryConfigurationException {
+		logger.info("Duration tree trainer started.");
+		FeatureFileReader featureFile = FeatureFileReader.getFeatureFileReader(getProp(FEATUREFILE));
+		UnitFileReader unitFile = new UnitFileReader(getProp(UNITFILE));
 
+		FeatureVector[] allFeatureVectors = featureFile.getFeatureVectors();
+		int maxData = Integer.parseInt(getProp(MAXDATA));
+		if (maxData == 0)
+			maxData = allFeatureVectors.length;
+		FeatureVector[] featureVectors = new FeatureVector[Math.min(maxData, allFeatureVectors.length)];
+		System.arraycopy(allFeatureVectors, 0, featureVectors, 0, featureVectors.length);
+		logger.debug("Total of " + allFeatureVectors.length + " feature vectors -- will use " + featureVectors.length);
 
-    public static void main(String[] args) throws Exception
-    {
-        DurationTreeTrainer dct = new DurationTreeTrainer();
-         DatabaseLayout db = new DatabaseLayout(dct);
-         dct.compute();
-    }
+		AgglomerativeClusterer clusterer = new AgglomerativeClusterer(featureVectors, featureFile.getFeatureDefinition(), null,
+				new DurationDistanceMeasure(unitFile), Float.parseFloat(getProp(PROPORTIONTESTDATA)));
+		DirectedGraphWriter writer = new DirectedGraphWriter();
+		DirectedGraph graph;
+		int iteration = 0;
+		do {
+			graph = clusterer.cluster();
+			iteration++;
+			if (graph != null) {
+				writer.saveGraph(graph, getProp(DURTREE) + ".level" + iteration);
+			}
+		} while (clusterer.canClusterMore());
 
+		if (graph == null) {
+			return false;
+		}
+
+		// Now replace each leaf with a FloatLeafNode containing mean and stddev
+		for (LeafNode leaf : graph.getLeafNodes()) {
+			FeatureVectorLeafNode fvLeaf = (FeatureVectorLeafNode) leaf;
+			FeatureVector[] fvs = fvLeaf.getFeatureVectors();
+			double[] dur = new double[fvs.length];
+			for (int i = 0; i < fvs.length; i++) {
+				dur[i] = unitFile.getUnit(fvs[i].getUnitIndex()).duration / (float) unitFile.getSampleRate();
+			}
+			double mean = MathUtils.mean(dur);
+			double stddev = MathUtils.standardDeviation(dur, mean);
+			FloatLeafNode floatLeaf = new FloatLeafNode(new float[] { (float) stddev, (float) mean });
+			Node mother = fvLeaf.getMother();
+			assert mother != null;
+			if (mother.isDecisionNode()) {
+				((DecisionNode) mother).replaceDaughter(floatLeaf, fvLeaf.getNodeIndex());
+			} else {
+				assert mother.isDirectedGraphNode();
+				assert ((DirectedGraphNode) mother).getLeafNode() == fvLeaf;
+				((DirectedGraphNode) mother).setLeafNode(floatLeaf);
+			}
+		}
+		writer.saveGraph(graph, getProp(DURTREE));
+		return true;
+
+	}
+
+	/**
+	 * Provide the progress of computation, in percent, or -1 if that feature is not implemented.
+	 * 
+	 * @return -1 if not implemented, or an integer between 0 and 100.
+	 */
+	public int getProgress() {
+		return -1;
+	}
+
+	public static void main(String[] args) throws Exception {
+		DurationTreeTrainer dct = new DurationTreeTrainer();
+		DatabaseLayout db = new DatabaseLayout(dct);
+		dct.compute();
+	}
 
 }
-

marytts-builder/src/main/java/marytts/tools/voiceimport/EndpointDetector.java

@@ -30,173 +30,155 @@ import marytts.util.data.audio.AudioConverterUtils;
 import marytts.util.io.BasenameList;
 import marytts.util.math.MathUtils;
 
-
-
 /**
- * Identify and Remove End-ponints (intitial and final silences) from
- * given set of wave files.
+ * Identify and Remove End-ponints (intitial and final silences) from given set of wave files.
+ * 
  * @author Sathish and Oytun
- *  
+ * 
  */
-public class EndpointDetector extends VoiceImportComponent
-{
-    protected File textDir;
-    protected File inputWavDir;
-    protected File outputWavDir;
-    protected String waveExt = ".wav";
-    private BasenameList bnlist;
+public class EndpointDetector extends VoiceImportComponent {
+	protected File textDir;
+	protected File inputWavDir;
+	protected File outputWavDir;
+	protected String waveExt = ".wav";
+	private BasenameList bnlist;
 
-    protected DatabaseLayout db = null;
-    protected int percent = 0;
+	protected DatabaseLayout db = null;
+	protected int percent = 0;
 
-    public String INPUTWAVDIR = "EndpointDetector.inputWaveDirectory";
-    public String OUTPUTWAVDIR = "EndpointDetector.outputWaveDirectory";
-    public String ENERGYBUFFERLENGTH = "EndpointDetector.energyBufferLength";
-    public String SPEECHSTARTLIKELIHOOD = "EndpointDetector.speechStartLikelihood";
-    public String SPEECHENDLIKELIHOOD = "EndpointDetector.speechEndLikelihood";
-    public String SHIFTFROMMINIMUMENERGYCENTER = "EndpointDetector.shiftFromMinimumEnergyCenter";
-    public String NUMENERGYCLUSTERS = "EndpointDetector.numEnergyClusters";
-    public String MINIMUMSTARTSILENCEINSECONDS = "EndpointDetector.minimumStartSilenceInSeconds";
-    public String MINIMUMENDSILENCEINSECONDS = "EndpointDetector.minimumEndSilenceInSeconds";
+	public String INPUTWAVDIR = "EndpointDetector.inputWaveDirectory";
+	public String OUTPUTWAVDIR = "EndpointDetector.outputWaveDirectory";
+	public String ENERGYBUFFERLENGTH = "EndpointDetector.energyBufferLength";
+	public String SPEECHSTARTLIKELIHOOD = "EndpointDetector.speechStartLikelihood";
+	public String SPEECHENDLIKELIHOOD = "EndpointDetector.speechEndLikelihood";
+	public String SHIFTFROMMINIMUMENERGYCENTER = "EndpointDetector.shiftFromMinimumEnergyCenter";
+	public String NUMENERGYCLUSTERS = "EndpointDetector.numEnergyClusters";
+	public String MINIMUMSTARTSILENCEINSECONDS = "EndpointDetector.minimumStartSilenceInSeconds";
+	public String MINIMUMENDSILENCEINSECONDS = "EndpointDetector.minimumEndSilenceInSeconds";
 
-    public String getName(){
-        return "EndpointDetector";
-    }
+	public String getName() {
+		return "EndpointDetector";
+	}
 
+	public SortedMap<String, String> getDefaultProps(DatabaseLayout theDb) {
+		this.db = theDb;
+		if (props == null) {
+			props = new TreeMap<String, String>();
+			props.put(INPUTWAVDIR, db.getProp(db.ROOTDIR) + "inputwav" + System.getProperty("file.separator"));
 
-    public SortedMap<String,String> getDefaultProps(DatabaseLayout theDb){
-        this.db = theDb;
-        if (props == null){
-            props = new TreeMap<String, String>();
-            props.put(INPUTWAVDIR, db.getProp(db.ROOTDIR)
-                    +"inputwav"
-                    +System.getProperty("file.separator"));
+			props.put(OUTPUTWAVDIR, db.getProp(db.ROOTDIR) + "outputwav" + System.getProperty("file.separator"));
 
-            props.put(OUTPUTWAVDIR, db.getProp(db.ROOTDIR)
-                    +"outputwav"
-                    +System.getProperty("file.separator"));
+			props.put(ENERGYBUFFERLENGTH, "20");
+			props.put(SPEECHSTARTLIKELIHOOD, "0.1");
+			props.put(SPEECHENDLIKELIHOOD, "0.1");
+			props.put(SHIFTFROMMINIMUMENERGYCENTER, "0.0");
+			props.put(NUMENERGYCLUSTERS, "4");
+			props.put(MINIMUMSTARTSILENCEINSECONDS, "1.0");
+			props.put(MINIMUMENDSILENCEINSECONDS, "1.0");
+		}
+		return props;
+	}
 
-            props.put(ENERGYBUFFERLENGTH, "20");
-            props.put(SPEECHSTARTLIKELIHOOD, "0.1");
-            props.put(SPEECHENDLIKELIHOOD, "0.1");
-            props.put(SHIFTFROMMINIMUMENERGYCENTER, "0.0");
-            props.put(NUMENERGYCLUSTERS, "4");
-            props.put(MINIMUMSTARTSILENCEINSECONDS, "1.0");
-            props.put(MINIMUMENDSILENCEINSECONDS, "1.0");
-        } 
-        return props;
-    }
+	protected void setupHelp() {
+		props2Help = new TreeMap<String, String>();
 
-    protected void setupHelp()
-    {
-        props2Help = new TreeMap<String, String>();
-        
-        props2Help.put(INPUTWAVDIR, "input wave files directory."); 
+		props2Help.put(INPUTWAVDIR, "input wave files directory.");
 
-        props2Help.put(OUTPUTWAVDIR, "output directory to store initial-end silences removed wave files." 
-                + "Will be created if it does not exist");
+		props2Help.put(OUTPUTWAVDIR, "output directory to store initial-end silences removed wave files."
+				+ "Will be created if it does not exist");
 
-        props2Help.put(ENERGYBUFFERLENGTH, "number of consecutive speech frames when searching for speech/silence start events" 
-                + "Range [1, 1000], decrease to detect more events");
-        
-        props2Help.put(SPEECHSTARTLIKELIHOOD, "likelihood of speech starting event"
-                + "Range [0.0,1.0], decrease to get more silence before speech segments");
-        
-        props2Help.put(SPEECHENDLIKELIHOOD, "likelihood of speech ending event"
-                + "Range [0.0,1.0], decrease to get more silence after speech segments");
-        
-        props2Help.put(SHIFTFROMMINIMUMENERGYCENTER, "multiplied by lowest energy cluster mean to generate speech/silence energy threshold"
-                + "Range [0.0,5.0], decrease to get more silence in speech segments");
-        
-        props2Help.put(NUMENERGYCLUSTERS, "number of energy clusters"
-                + "Range [1,20], decrease to get more silence in speech segments");
-        
-        props2Help.put(MINIMUMSTARTSILENCEINSECONDS, "minimum silence in the beginning of the output files in seconds"
-                + "Range [0.0,30.0], increase to get more silence in the beginning");
-        
-        props2Help.put(MINIMUMENDSILENCEINSECONDS, "minimum silence at the end of the output files in seconds"
-                + "Range [0.0,30.0], increase to get more silence at the end");
-    }
+		props2Help.put(ENERGYBUFFERLENGTH, "number of consecutive speech frames when searching for speech/silence start events"
+				+ "Range [1, 1000], decrease to detect more events");
 
+		props2Help.put(SPEECHSTARTLIKELIHOOD, "likelihood of speech starting event"
+				+ "Range [0.0,1.0], decrease to get more silence before speech segments");
 
-    public boolean compute() throws IOException, UnsupportedAudioFileException
-    {
-        //      Check existance of input directory  
-        inputWavDir = new File(getProp(INPUTWAVDIR));
-        if (!inputWavDir.exists()){
-            throw new Error("Could not find input Directory: "+ getProp(INPUTWAVDIR));
-        }   
+		props2Help.put(SPEECHENDLIKELIHOOD, "likelihood of speech ending event"
+				+ "Range [0.0,1.0], decrease to get more silence after speech segments");
 
-        // Check existance of output directory
-        // if not exists, create a new directory
-        outputWavDir = new File(getProp(OUTPUTWAVDIR));
-        if (!outputWavDir.exists()){
-            System.out.print(OUTPUTWAVDIR+" "+getProp(OUTPUTWAVDIR)
-                    +" does not exist; ");
-            if (!outputWavDir.mkdir()){
-                throw new Error("Could not create OUTPUTWAVDIR");
-            }
-            System.out.print("Created successfully.\n");
-        }
+		props2Help.put(SHIFTFROMMINIMUMENERGYCENTER,
+				"multiplied by lowest energy cluster mean to generate speech/silence energy threshold"
+						+ "Range [0.0,5.0], decrease to get more silence in speech segments");
 
-        // Automatically collect all ".wav" files from given directory
-        bnlist = new BasenameList(inputWavDir+File.separator,waveExt);
+		props2Help.put(NUMENERGYCLUSTERS, "number of energy clusters"
+				+ "Range [1,20], decrease to get more silence in speech segments");
 
+		props2Help.put(MINIMUMSTARTSILENCEINSECONDS, "minimum silence in the beginning of the output files in seconds"
+				+ "Range [0.0,30.0], increase to get more silence in the beginning");
 
-        int energyBufferLength = Integer.valueOf(getProp(ENERGYBUFFERLENGTH));
-        energyBufferLength = MathUtils.CheckLimits(energyBufferLength, 1, 1000);
-        
-        double speechStartLikelihood = Double.valueOf(getProp(SPEECHSTARTLIKELIHOOD));
-        speechStartLikelihood = MathUtils.CheckLimits(speechStartLikelihood, 0.0, 1.0);
-        
-        double speechEndLikelihood = Double.valueOf(getProp(SPEECHENDLIKELIHOOD));
-        speechEndLikelihood = MathUtils.CheckLimits(speechEndLikelihood, 0.0, 1.0);
-        
-        double shiftFromMinimumEnergyCenter = Double.valueOf(getProp(SHIFTFROMMINIMUMENERGYCENTER));
-        shiftFromMinimumEnergyCenter = MathUtils.CheckLimits(shiftFromMinimumEnergyCenter, 0.0, 5.0);
-        
-        int numClusters = Integer.valueOf(getProp(NUMENERGYCLUSTERS));
-        numClusters = MathUtils.CheckLimits(numClusters, 1, 20);
-        
-        double minimumStartSilenceInSeconds = Double.valueOf(getProp(MINIMUMSTARTSILENCEINSECONDS));
-        minimumStartSilenceInSeconds = MathUtils.CheckLimits(minimumStartSilenceInSeconds, 0.0, 30.0);
-        
-        double minimumEndSilenceInSeconds = Double.valueOf(getProp(MINIMUMENDSILENCEINSECONDS));
-        minimumEndSilenceInSeconds = MathUtils.CheckLimits(minimumEndSilenceInSeconds, 0.0, 30.0);
-        //
+		props2Help.put(MINIMUMENDSILENCEINSECONDS, "minimum silence at the end of the output files in seconds"
+				+ "Range [0.0,30.0], increase to get more silence at the end");
+	}
 
-        System.out.println( "Removing endpoints for "+ bnlist.getLength() + " wave files" );
+	public boolean compute() throws IOException, UnsupportedAudioFileException {
+		// Check existance of input directory
+		inputWavDir = new File(getProp(INPUTWAVDIR));
+		if (!inputWavDir.exists()) {
+			throw new Error("Could not find input Directory: " + getProp(INPUTWAVDIR));
+		}
 
-        for (int i=0; i<bnlist.getLength(); i++) 
-        {
-            percent = 100*i/bnlist.getLength();
-            String inputFile = inputWavDir + File.separator + bnlist.getName(i) + waveExt;
-            String outputFile = outputWavDir + File.separator + bnlist.getName(i) + waveExt;
+		// Check existance of output directory
+		// if not exists, create a new directory
+		outputWavDir = new File(getProp(OUTPUTWAVDIR));
+		if (!outputWavDir.exists()) {
+			System.out.print(OUTPUTWAVDIR + " " + getProp(OUTPUTWAVDIR) + " does not exist; ");
+			if (!outputWavDir.mkdir()) {
+				throw new Error("Could not create OUTPUTWAVDIR");
+			}
+			System.out.print("Created successfully.\n");
+		}
 
-            AudioConverterUtils.removeEndpoints(inputFile, outputFile,
-                    energyBufferLength, speechStartLikelihood, speechEndLikelihood, shiftFromMinimumEnergyCenter, numClusters,
-                    minimumStartSilenceInSeconds, minimumEndSilenceInSeconds);
+		// Automatically collect all ".wav" files from given directory
+		bnlist = new BasenameList(inputWavDir + File.separator, waveExt);
 
-            System.out.println( "    " + bnlist.getName(i) );
-        }
+		int energyBufferLength = Integer.valueOf(getProp(ENERGYBUFFERLENGTH));
+		energyBufferLength = MathUtils.CheckLimits(energyBufferLength, 1, 1000);
 
-        System.out.println("...Done.");
+		double speechStartLikelihood = Double.valueOf(getProp(SPEECHSTARTLIKELIHOOD));
+		speechStartLikelihood = MathUtils.CheckLimits(speechStartLikelihood, 0.0, 1.0);
 
-        return true;
-    }
+		double speechEndLikelihood = Double.valueOf(getProp(SPEECHENDLIKELIHOOD));
+		speechEndLikelihood = MathUtils.CheckLimits(speechEndLikelihood, 0.0, 1.0);
 
+		double shiftFromMinimumEnergyCenter = Double.valueOf(getProp(SHIFTFROMMINIMUMENERGYCENTER));
+		shiftFromMinimumEnergyCenter = MathUtils.CheckLimits(shiftFromMinimumEnergyCenter, 0.0, 5.0);
 
+		int numClusters = Integer.valueOf(getProp(NUMENERGYCLUSTERS));
+		numClusters = MathUtils.CheckLimits(numClusters, 1, 20);
 
+		double minimumStartSilenceInSeconds = Double.valueOf(getProp(MINIMUMSTARTSILENCEINSECONDS));
+		minimumStartSilenceInSeconds = MathUtils.CheckLimits(minimumStartSilenceInSeconds, 0.0, 30.0);
 
-    /**
-     * Provide the progress of computation, in percent, or -1 if
-     * that feature is not implemented.
-     * @return -1 if not implemented, or an integer between 0 and 100.
-     */
-    public int getProgress()
-    {
-        return percent;
-    }
+		double minimumEndSilenceInSeconds = Double.valueOf(getProp(MINIMUMENDSILENCEINSECONDS));
+		minimumEndSilenceInSeconds = MathUtils.CheckLimits(minimumEndSilenceInSeconds, 0.0, 30.0);
+		//
+
+		System.out.println("Removing endpoints for " + bnlist.getLength() + " wave files");
+
+		for (int i = 0; i < bnlist.getLength(); i++) {
+			percent = 100 * i / bnlist.getLength();
+			String inputFile = inputWavDir + File.separator + bnlist.getName(i) + waveExt;
+			String outputFile = outputWavDir + File.separator + bnlist.getName(i) + waveExt;
+
+			AudioConverterUtils.removeEndpoints(inputFile, outputFile, energyBufferLength, speechStartLikelihood,
+					speechEndLikelihood, shiftFromMinimumEnergyCenter, numClusters, minimumStartSilenceInSeconds,
+					minimumEndSilenceInSeconds);
+
+			System.out.println("    " + bnlist.getName(i));
+		}
+
+		System.out.println("...Done.");
+
+		return true;
+	}
+
+	/**
+	 * Provide the progress of computation, in percent, or -1 if that feature is not implemented.
+	 * 
+	 * @return -1 if not implemented, or an integer between 0 and 100.
+	 */
+	public int getProgress() {
+		return percent;
+	}
 
 }
-

marytts-builder/src/main/java/marytts/tools/voiceimport/F0CARTTrainer.java

@@ -48,402 +48,390 @@ import marytts.unitselection.data.UnitFileReader;
 import marytts.util.data.Datagram;
 import marytts.util.string.PrintfFormat;
 
-
 /**
- * A class which converts a text file in festvox format
- * into a one-file-per-utterance format in a given directory.
+ * A class which converts a text file in festvox format into a one-file-per-utterance format in a given directory.
+ * 
  * @author schroed
  *
  */
-public class F0CARTTrainer extends VoiceImportComponent
-{
-    protected File f0Dir;
-    protected File leftF0FeaturesFile;
-    protected File midF0FeaturesFile;
-    protected File rightF0FeaturesFile;
-    protected File f0DescFile;
-    protected File wagonLeftTreeFile;
-    protected File wagonMidTreeFile;
-    protected File wagonRightTreeFile;
-    protected String featureExt = ".pfeats";  
-    protected DatabaseLayout db = null;
-    protected int percent = 0;
-    protected boolean useStepwiseTraining = false;
-    
-    private final String name = "F0CARTTrainer";
-    public final String STEPWISETRAINING = name+".stepwiseTraining";
-    public final String FEATUREFILE = name+".featureFile";
-    public final String UNITFILE = name+".unitFile";
-    public final String WAVETIMELINE = name+".waveTimeline";
-    public final String ISHNMTIMELINE = name+".isHnmTimeline";
-    public final String F0LEFTTREEFILE = name+".f0LeftTreeFile";
-    public final String F0RIGHTTREEFILE = name+".f0RightTreeFile";
-    public final String F0MIDTREEFILE = name+".f0MidTreeFile";
-    
-    public F0CARTTrainer(){
-        setupHelp();
-    }
-    
-    public String getName(){
-        return name;
-    }
+public class F0CARTTrainer extends VoiceImportComponent {
+	protected File f0Dir;
+	protected File leftF0FeaturesFile;
+	protected File midF0FeaturesFile;
+	protected File rightF0FeaturesFile;
+	protected File f0DescFile;
+	protected File wagonLeftTreeFile;
+	protected File wagonMidTreeFile;
+	protected File wagonRightTreeFile;
+	protected String featureExt = ".pfeats";
+	protected DatabaseLayout db = null;
+	protected int percent = 0;
+	protected boolean useStepwiseTraining = false;
 
-    @Override
-    protected void initialiseComp()
-    {       
-        String rootDir = db.getProp(DatabaseLayout.ROOTDIR);
-        String f0DirName = db.getProp(DatabaseLayout.TEMPDIR);
-        this.f0Dir = new File(f0DirName);
-        if (!f0Dir.exists()){
-            System.out.print("temp dir "+f0DirName
-                    +" does not exist; ");
-            if (!f0Dir.mkdir()){
-                throw new Error("Could not create F0DIR");
-            }
-            System.out.print("Created successfully.\n");
-        }  
-        this.leftF0FeaturesFile = new File(f0Dir, "f0.left.feats");
-        this.midF0FeaturesFile = new File(f0Dir, "f0.mid.feats");
-        this.rightF0FeaturesFile = new File(f0Dir, "f0.right.feats");
-        this.f0DescFile = new File(f0Dir, "f0.desc");
-        this.wagonLeftTreeFile = new File(f0Dir, "f0.left.tree");
-        this.wagonMidTreeFile = new File(f0Dir, "f0.mid.tree");
-        this.wagonRightTreeFile = new File(f0Dir, "f0.right.tree");
-        this.useStepwiseTraining = Boolean.valueOf(getProp(STEPWISETRAINING)).booleanValue();
-    }
-     
-     public SortedMap<String, String> getDefaultProps(DatabaseLayout dbl){
-        this.db = dbl;
-       if (props == null){
-           props = new TreeMap<String, String>();
-           String filedir = db.getProp(DatabaseLayout.FILEDIR);
-           String maryext = db.getProp(DatabaseLayout.MARYEXT);
-           props.put(STEPWISETRAINING,"false");
-           props.put(FEATUREFILE, filedir+"phoneFeatures"+maryext);
-           props.put(UNITFILE, filedir+"phoneUnits"+maryext);
-           props.put(WAVETIMELINE, db.getProp(DatabaseLayout.FILEDIR)
-                        +"timeline_waveforms"+db.getProp(DatabaseLayout.MARYEXT));      
-           props.put(ISHNMTIMELINE, "false"); 
-           props.put(F0LEFTTREEFILE,filedir+"f0.left.tree");
-           props.put(F0RIGHTTREEFILE,filedir+"f0.right.tree");
-           props.put(F0MIDTREEFILE,filedir+"f0.mid.tree");
-       }
-       return props;
-     }
-     
-     protected void setupHelp(){         
-         props2Help = new TreeMap<String, String>();
-         props2Help.put(STEPWISETRAINING,"\"false\" or \"true\" ????????????????????????");
-         props2Help.put(FEATUREFILE, "file containing all phone units and their target cost features");
-         props2Help.put(UNITFILE, "file containing all phone units");
-         props2Help.put(WAVETIMELINE, "file containing all waveforms or models that can genarate them"); 
-         props2Help.put(ISHNMTIMELINE, "file containing all wave files");         
-         props2Help.put(F0LEFTTREEFILE,"file containing the left f0 CART. Will be created by this module");
-         props2Help.put(F0RIGHTTREEFILE,"file containing the right f0 CART. Will be created by this module");
-         props2Help.put(F0MIDTREEFILE,"file containing the middle f0 CART. Will be created by this module");
-     }
-   
+	private final String name = "F0CARTTrainer";
+	public final String STEPWISETRAINING = name + ".stepwiseTraining";
+	public final String FEATUREFILE = name + ".featureFile";
+	public final String UNITFILE = name + ".unitFile";
+	public final String WAVETIMELINE = name + ".waveTimeline";
+	public final String ISHNMTIMELINE = name + ".isHnmTimeline";
+	public final String F0LEFTTREEFILE = name + ".f0LeftTreeFile";
+	public final String F0RIGHTTREEFILE = name + ".f0RightTreeFile";
+	public final String F0MIDTREEFILE = name + ".f0MidTreeFile";
 
-     
-     @Override
-     public boolean compute() throws IOException, MaryConfigurationException
-     {
-        FeatureFileReader featureFile = 
-            FeatureFileReader.getFeatureFileReader(getProp(FEATUREFILE));
-        UnitFileReader unitFile = new UnitFileReader(getProp(UNITFILE));
-        TimelineReader waveTimeline = null;
-        if (getProp(ISHNMTIMELINE).compareToIgnoreCase("true")==0)
-            waveTimeline = new HnmTimelineReader(getProp(WAVETIMELINE));
-        else
-            waveTimeline = new TimelineReader(getProp(WAVETIMELINE));
-        
-        PrintWriter toLeftFeaturesFile = 
-            new PrintWriter(new FileOutputStream(leftF0FeaturesFile));
-        PrintWriter toMidFeaturesFile =
-            new PrintWriter(new FileOutputStream(midF0FeaturesFile));
-        PrintWriter toRightFeaturesFile = 
-            new PrintWriter(new FileOutputStream(rightF0FeaturesFile));
+	public F0CARTTrainer() {
+		setupHelp();
+	}
 
-        System.out.println("F0 CART trainer: exporting f0 features");
-        
-        FeatureDefinition featureDefinition = featureFile.getFeatureDefinition();
-        byte isVowel = featureDefinition.getFeatureValueAsByte("ph_vc", "+");
-        int iVC = featureDefinition.getFeatureIndex("ph_vc");
-        int iSegsFromSylStart = featureDefinition.getFeatureIndex("segs_from_syl_start");
-        int iSegsFromSylEnd = featureDefinition.getFeatureIndex("segs_from_syl_end");
-        int iCVoiced = featureDefinition.getFeatureIndex("ph_cvox");
-        byte isCVoiced = featureDefinition.getFeatureValueAsByte("ph_cvox", "+");
+	public String getName() {
+		return name;
+	}
 
-        int nSyllables = 0;
-        for (int i=0, len=unitFile.getNumberOfUnits(); i<len; i++) {
-            // We estimate that feature extraction takes 1/10 of the total time
-            // (that's probably wrong, but never mind)
-            percent = 10*i/len;
-            FeatureVector fv = featureFile.getFeatureVector(i);
-            if (fv.getByteFeature(iVC) == isVowel) {
-                // Found a vowel, i.e. found a syllable.
-                int mid = i;
-                FeatureVector fvMid = fv;
-                // Now find first/last voiced unit in the syllable:
-                int first = i;
-                for(int j=1, lookLeft = (int)fvMid.getByteFeature(iSegsFromSylStart); j<lookLeft; j++) {
-                    fv = featureFile.getFeatureVector(mid-j); // units are in sequential order
-                    // No need to check if there are any vowels to the left of this one,
-                    // because of the way we do the search in the top-level loop.
-                    if (fv.getByteFeature(iCVoiced) != isCVoiced) {
-                        break; // mid-j is not voiced
-                    }
-                    first = mid-j; // OK, the unit we are currently looking at is part of the voiced syllable section
-                }
-                int last = i;
-                for(int j=1, lookRight = (int)fvMid.getByteFeature(iSegsFromSylEnd); j<lookRight; j++) {
-                    fv = featureFile.getFeatureVector(mid+j); // units are in sequential order
+	@Override
+	protected void initialiseComp() {
+		String rootDir = db.getProp(DatabaseLayout.ROOTDIR);
+		String f0DirName = db.getProp(DatabaseLayout.TEMPDIR);
+		this.f0Dir = new File(f0DirName);
+		if (!f0Dir.exists()) {
+			System.out.print("temp dir " + f0DirName + " does not exist; ");
+			if (!f0Dir.mkdir()) {
+				throw new Error("Could not create F0DIR");
+			}
+			System.out.print("Created successfully.\n");
+		}
+		this.leftF0FeaturesFile = new File(f0Dir, "f0.left.feats");
+		this.midF0FeaturesFile = new File(f0Dir, "f0.mid.feats");
+		this.rightF0FeaturesFile = new File(f0Dir, "f0.right.feats");
+		this.f0DescFile = new File(f0Dir, "f0.desc");
+		this.wagonLeftTreeFile = new File(f0Dir, "f0.left.tree");
+		this.wagonMidTreeFile = new File(f0Dir, "f0.mid.tree");
+		this.wagonRightTreeFile = new File(f0Dir, "f0.right.tree");
+		this.useStepwiseTraining = Boolean.valueOf(getProp(STEPWISETRAINING)).booleanValue();
+	}
 
-                    if (fv.getByteFeature(iVC) != isVowel && fv.getByteFeature(iCVoiced) != isCVoiced) {
-                        break; // mid+j is not voiced
-                    }
-                    last = mid+j; // OK, the unit we are currently looking at is part of the voiced syllable section
-                }
-                // TODO: make this more robust, e.g. by fitting two straight lines to the data:
-                Datagram[] midDatagrams = waveTimeline.getDatagrams(unitFile.getUnit(mid), unitFile.getSampleRate());
-                Datagram[] leftDatagrams = waveTimeline.getDatagrams(unitFile.getUnit(first), unitFile.getSampleRate());
-                Datagram[] rightDatagrams = waveTimeline.getDatagrams(unitFile.getUnit(last), unitFile.getSampleRate());
-                if (midDatagrams != null && midDatagrams.length > 0
-                        && leftDatagrams != null && leftDatagrams.length > 0
-                        && rightDatagrams != null && rightDatagrams.length > 0) {
-                    float midF0 = waveTimeline.getSampleRate() / (float) midDatagrams[midDatagrams.length/2].getDuration();
-                    float leftF0 = waveTimeline.getSampleRate() / (float) leftDatagrams[0].getDuration();
-                    float rightF0 = waveTimeline.getSampleRate() / (float) rightDatagrams[rightDatagrams.length-1].getDuration();
-                    System.out.println("Syllable at "+mid+" (length "+(last-first+1)+"): left = "+((int)leftF0)+", mid = "+((int)midF0)+", right = "+rightF0);
-                    toLeftFeaturesFile.println(leftF0 + " "+ featureDefinition.toFeatureString(fvMid));
-                    toMidFeaturesFile.println(midF0 + " "+ featureDefinition.toFeatureString(fvMid));
-                    toRightFeaturesFile.println(rightF0 + " "+ featureDefinition.toFeatureString(fvMid));
-                    nSyllables++;
-                    
-                }                 
-                // Skip the part we just covered:
-                i = last;               
-            }
-        }
-        toLeftFeaturesFile.close();
-        toMidFeaturesFile.close();
-        toRightFeaturesFile.close();
-        System.out.println("F0 features extracted for "+nSyllables+" syllables");
-        
-        if (useStepwiseTraining) percent = 1; // estimated
-        else percent = 10; // estimated
-        PrintWriter toDesc = new PrintWriter(new FileOutputStream(f0DescFile));
-        generateFeatureDescriptionForWagon(featureDefinition, toDesc);
-        toDesc.close();
-        
-        // Now, call wagon
-        WagonCaller wagonCaller = new WagonCaller(db.getProp(DatabaseLayout.ESTDIR), null);
-        boolean ok;
-        if (useStepwiseTraining) {
-            // Split the data set in training and test part:
-            // TODO: hardcoded path = EVIL
-            Process traintest = Runtime.getRuntime().exec("/project/mary/Festival/festvox/src/general/traintest "+leftF0FeaturesFile.getAbsolutePath());
-            try {
-                traintest.waitFor();
-            } catch (InterruptedException ie) {}
-            ok = wagonCaller.callWagon("-data "+leftF0FeaturesFile.getAbsolutePath()+".train"
-                    +" -test "+leftF0FeaturesFile.getAbsolutePath()+".test -stepwise"
-                    +" -desc "+f0DescFile.getAbsolutePath()
-                    +" -stop 10 "
-                    +" -output "+wagonLeftTreeFile.getAbsolutePath());
-        } else {
-            ok = wagonCaller.callWagon("-data "+leftF0FeaturesFile.getAbsolutePath()
-                    +" -desc "+f0DescFile.getAbsolutePath()
-                    +" -stop 10 "
-                    +" -output "+wagonLeftTreeFile.getAbsolutePath());
-        }
-        if (!ok) return false;
-        percent = 40;
-        if (useStepwiseTraining) {
-            // Split the data set in training and test part:
-            // hardcoded path = EVIL
-            Process traintest = Runtime.getRuntime().exec("/project/mary/Festival/festvox/src/general/traintest "+midF0FeaturesFile.getAbsolutePath());
-            try {
-                traintest.waitFor();
-            } catch (InterruptedException ie) {}
-            ok = wagonCaller.callWagon("-data "+midF0FeaturesFile.getAbsolutePath()+".train"
-                    +" -test "+midF0FeaturesFile.getAbsolutePath()+".test -stepwise"
-                    +" -desc "+f0DescFile.getAbsolutePath()
-                    +" -stop 10 "
-                    +" -output "+wagonMidTreeFile.getAbsolutePath());
-        } else {
-            ok = wagonCaller.callWagon("-data "+midF0FeaturesFile.getAbsolutePath()
-                    +" -desc "+f0DescFile.getAbsolutePath()
-                    +" -stop 10 "
-                    +" -output "+wagonMidTreeFile.getAbsolutePath());
-        }
-        if (!ok) return false;
-        percent = 70;
-        if (useStepwiseTraining) {
-            // Split the data set in training and test part:
-            // hardcoded path = EVIL
-            Process traintest = Runtime.getRuntime().exec("/project/mary/Festival/festvox/src/general/traintest "+rightF0FeaturesFile.getAbsolutePath());
-            try {
-                traintest.waitFor();
-            } catch (InterruptedException ie) {}
-            ok = wagonCaller.callWagon("-data "+rightF0FeaturesFile.getAbsolutePath()+".train"
-                    +" -test "+rightF0FeaturesFile.getAbsolutePath()+".test -stepwise"
-                    +" -desc "+f0DescFile.getAbsolutePath()
-                    +" -stop 10 "
-                    +" -output "+wagonRightTreeFile.getAbsolutePath());
-        } else {
-            ok = wagonCaller.callWagon("-data "+rightF0FeaturesFile.getAbsolutePath()
-                    +" -desc "+f0DescFile.getAbsolutePath()
-                    +" -stop 10 "
-                    +" -output "+wagonRightTreeFile.getAbsolutePath());
-        }
-        
-        if(ok){
-            
-            // F0 Left file
-            String destinationFile = getProp(F0LEFTTREEFILE);
-            WagonCARTReader wagonLReader = new WagonCARTReader(LeafType.FloatLeafNode);
-            marytts.cart.Node rootLNode = wagonLReader.load(new BufferedReader(new FileReader(wagonLeftTreeFile)), featureDefinition);
-            CART leftF0Cart = new CART(rootLNode, featureDefinition);
-            MaryCARTWriter wwl = new MaryCARTWriter();
-            wwl.dumpMaryCART(leftF0Cart, destinationFile);
-            
-            // F0 Mid tree
-            destinationFile = getProp(F0MIDTREEFILE);
-            WagonCARTReader wagonMReader = new WagonCARTReader(LeafType.FloatLeafNode);
-            marytts.cart.Node rootMNode = wagonMReader.load(new BufferedReader(new FileReader(wagonMidTreeFile)), featureDefinition);
-            CART midF0Cart = new CART(rootMNode, featureDefinition);
-            MaryCARTWriter wwm = new MaryCARTWriter();
-            wwm.dumpMaryCART(midF0Cart, destinationFile);
-            
-            // F0 Right tree
-            destinationFile = getProp(F0RIGHTTREEFILE);
-            WagonCARTReader wagonRReader = new WagonCARTReader(LeafType.FloatLeafNode);
-            marytts.cart.Node rootRNode = wagonRReader.load(new BufferedReader(new FileReader(wagonRightTreeFile)), featureDefinition);
-            CART rightF0Cart = new CART(rootRNode, featureDefinition);
-            MaryCARTWriter wwr = new MaryCARTWriter();
-            wwr.dumpMaryCART(rightF0Cart, destinationFile);
-            
-            
-        }
-        
-        percent = 100;
+	public SortedMap<String, String> getDefaultProps(DatabaseLayout dbl) {
+		this.db = dbl;
+		if (props == null) {
+			props = new TreeMap<String, String>();
+			String filedir = db.getProp(DatabaseLayout.FILEDIR);
+			String maryext = db.getProp(DatabaseLayout.MARYEXT);
+			props.put(STEPWISETRAINING, "false");
+			props.put(FEATUREFILE, filedir + "phoneFeatures" + maryext);
+			props.put(UNITFILE, filedir + "phoneUnits" + maryext);
+			props.put(WAVETIMELINE,
+					db.getProp(DatabaseLayout.FILEDIR) + "timeline_waveforms" + db.getProp(DatabaseLayout.MARYEXT));
+			props.put(ISHNMTIMELINE, "false");
+			props.put(F0LEFTTREEFILE, filedir + "f0.left.tree");
+			props.put(F0RIGHTTREEFILE, filedir + "f0.right.tree");
+			props.put(F0MIDTREEFILE, filedir + "f0.mid.tree");
+		}
+		return props;
+	}
 
-        return ok;
-    }
-    
-    private String[] align(String basename) throws IOException
-    {
-        BufferedReader labels = 
-            new BufferedReader(new InputStreamReader(new FileInputStream(
-            		new File(db.getProp(DatabaseLayout.PHONELABDIR), basename + DatabaseLayout.LABEXT)), "UTF-8"));
-        BufferedReader features = 
-            new BufferedReader(new InputStreamReader(new FileInputStream(
-            		new File(db.getProp(DatabaseLayout.PHONEFEATUREDIR), basename + featureExt)), "UTF-8")); 
-        String line;
-        // Skip label file header:
-        while ((line = labels.readLine()) != null) {
-            if (line.startsWith("#")) break; // line starting with "#" marks end of header
-        }
-        // Skip features file header:
-        while ((line = features.readLine()) != null) {
-            if (line.trim().equals("")) break; // empty line marks end of header
-        }
-        
-        // Now go through all feature file units
-        boolean correct = true;
-        int unitIndex = -1;
-        float prevEnd = 0;
-        List<String> aligned = new ArrayList<String>();
-        while (correct) {
-            unitIndex++;
-            String labelLine = labels.readLine();
-            String featureLine = features.readLine();
-            if (featureLine == null) { // incomplete feature file
-                return null;
-            } else if (featureLine.trim().equals("")) { // end of features
-                if (labelLine == null) break; // normal end found
-                else // label file is longer than feature file
-                    return null;
-            }
-            // Verify that the two labels are the same:
-            StringTokenizer st = new StringTokenizer(labelLine.trim());
-            // The third token in each line is the label
-            float end = Float.parseFloat(st.nextToken());
-            st.nextToken(); // skip
-            String labelUnit = st.nextToken();
+	protected void setupHelp() {
+		props2Help = new TreeMap<String, String>();
+		props2Help.put(STEPWISETRAINING, "\"false\" or \"true\" ????????????????????????");
+		props2Help.put(FEATUREFILE, "file containing all phone units and their target cost features");
+		props2Help.put(UNITFILE, "file containing all phone units");
+		props2Help.put(WAVETIMELINE, "file containing all waveforms or models that can genarate them");
+		props2Help.put(ISHNMTIMELINE, "file containing all wave files");
+		props2Help.put(F0LEFTTREEFILE, "file containing the left f0 CART. Will be created by this module");
+		props2Help.put(F0RIGHTTREEFILE, "file containing the right f0 CART. Will be created by this module");
+		props2Help.put(F0MIDTREEFILE, "file containing the middle f0 CART. Will be created by this module");
+	}
 
-            st = new StringTokenizer(featureLine.trim());
-            // The expect that the first token in each line is the label
-            String featureUnit = st.nextToken();
-            if (!featureUnit.equals(labelUnit)) {
-                // Non-matching units found
-                return null;
-            }
-            // OK, now we assume we have two matching lines.
-            if (!featureUnit.startsWith("_")) { // discard all silences
-                // Output format: unit duration, followed by the feature line
-                aligned.add(new PrintfFormat(Locale.ENGLISH, "%.3f").sprintf(end-prevEnd)+" "+featureLine.trim());
-            }
-            prevEnd = end;
-        }
-        return (String[]) aligned.toArray(new String[0]);
-    }
-    
-    private void generateFeatureDescriptionForWagon(FeatureDefinition fd, PrintWriter out)
-    {
-        out.println("(");
-        out.println("(f0 float)");
-        int nDiscreteFeatures = fd.getNumberOfByteFeatures()+fd.getNumberOfShortFeatures();
-        for (int i=0, n=fd.getNumberOfFeatures(); i<n; i++) {            
-            out.print("( ");
-            out.print(fd.getFeatureName(i));
-            if (i<nDiscreteFeatures) { // list values
-                if (fd.getNumberOfValues(i) == 20 && fd.getFeatureValueAsString(i, 19).equals("19")) {
-                    // one of our pseudo-floats
-                    out.println(" float )");
-                } else { // list the values
-                    for (int v=0, vmax=fd.getNumberOfValues(i); v<vmax; v++) {
-                        out.print("  ");
-                        String val = fd.getFeatureValueAsString(i, v);
-                        if (val.indexOf('"') != -1) {
-                            StringBuilder buf = new StringBuilder();
-                            for (int c=0; c<val.length(); c++) {
-                                char ch = val.charAt(c);
-                                if (ch == '"') buf.append("\\\"");
-                                else buf.append(ch);
-                            }
-                            val = buf.toString();
-                        }
-                        out.print("\""+val+"\"");
-                    }
-                    out.println(" )");
-                }
-            } else { // float feature
-                    out.println(" float )");
-            }
+	@Override
+	public boolean compute() throws IOException, MaryConfigurationException {
+		FeatureFileReader featureFile = FeatureFileReader.getFeatureFileReader(getProp(FEATUREFILE));
+		UnitFileReader unitFile = new UnitFileReader(getProp(UNITFILE));
+		TimelineReader waveTimeline = null;
+		if (getProp(ISHNMTIMELINE).compareToIgnoreCase("true") == 0)
+			waveTimeline = new HnmTimelineReader(getProp(WAVETIMELINE));
+		else
+			waveTimeline = new TimelineReader(getProp(WAVETIMELINE));
 
-        }
-        out.println(")");
-    }
+		PrintWriter toLeftFeaturesFile = new PrintWriter(new FileOutputStream(leftF0FeaturesFile));
+		PrintWriter toMidFeaturesFile = new PrintWriter(new FileOutputStream(midF0FeaturesFile));
+		PrintWriter toRightFeaturesFile = new PrintWriter(new FileOutputStream(rightF0FeaturesFile));
 
-    
-    /**
-     * Provide the progress of computation, in percent, or -1 if
-     * that feature is not implemented.
-     * @return -1 if not implemented, or an integer between 0 and 100.
-     */
-    public int getProgress()
-    {
-        return percent;
-    }
+		System.out.println("F0 CART trainer: exporting f0 features");
 
-    public static void main(String[] args) throws Exception
-    {
-        F0CARTTrainer f0ct = new F0CARTTrainer();
-        DatabaseLayout db = new DatabaseLayout(f0ct);
-        f0ct.compute();
-    }
+		FeatureDefinition featureDefinition = featureFile.getFeatureDefinition();
+		byte isVowel = featureDefinition.getFeatureValueAsByte("ph_vc", "+");
+		int iVC = featureDefinition.getFeatureIndex("ph_vc");
+		int iSegsFromSylStart = featureDefinition.getFeatureIndex("segs_from_syl_start");
+		int iSegsFromSylEnd = featureDefinition.getFeatureIndex("segs_from_syl_end");
+		int iCVoiced = featureDefinition.getFeatureIndex("ph_cvox");
+		byte isCVoiced = featureDefinition.getFeatureValueAsByte("ph_cvox", "+");
+
+		int nSyllables = 0;
+		for (int i = 0, len = unitFile.getNumberOfUnits(); i < len; i++) {
+			// We estimate that feature extraction takes 1/10 of the total time
+			// (that's probably wrong, but never mind)
+			percent = 10 * i / len;
+			FeatureVector fv = featureFile.getFeatureVector(i);
+			if (fv.getByteFeature(iVC) == isVowel) {
+				// Found a vowel, i.e. found a syllable.
+				int mid = i;
+				FeatureVector fvMid = fv;
+				// Now find first/last voiced unit in the syllable:
+				int first = i;
+				for (int j = 1, lookLeft = (int) fvMid.getByteFeature(iSegsFromSylStart); j < lookLeft; j++) {
+					fv = featureFile.getFeatureVector(mid - j); // units are in sequential order
+					// No need to check if there are any vowels to the left of this one,
+					// because of the way we do the search in the top-level loop.
+					if (fv.getByteFeature(iCVoiced) != isCVoiced) {
+						break; // mid-j is not voiced
+					}
+					first = mid - j; // OK, the unit we are currently looking at is part of the voiced syllable section
+				}
+				int last = i;
+				for (int j = 1, lookRight = (int) fvMid.getByteFeature(iSegsFromSylEnd); j < lookRight; j++) {
+					fv = featureFile.getFeatureVector(mid + j); // units are in sequential order
+
+					if (fv.getByteFeature(iVC) != isVowel && fv.getByteFeature(iCVoiced) != isCVoiced) {
+						break; // mid+j is not voiced
+					}
+					last = mid + j; // OK, the unit we are currently looking at is part of the voiced syllable section
+				}
+				// TODO: make this more robust, e.g. by fitting two straight lines to the data:
+				Datagram[] midDatagrams = waveTimeline.getDatagrams(unitFile.getUnit(mid), unitFile.getSampleRate());
+				Datagram[] leftDatagrams = waveTimeline.getDatagrams(unitFile.getUnit(first), unitFile.getSampleRate());
+				Datagram[] rightDatagrams = waveTimeline.getDatagrams(unitFile.getUnit(last), unitFile.getSampleRate());
+				if (midDatagrams != null && midDatagrams.length > 0 && leftDatagrams != null && leftDatagrams.length > 0
+						&& rightDatagrams != null && rightDatagrams.length > 0) {
+					float midF0 = waveTimeline.getSampleRate() / (float) midDatagrams[midDatagrams.length / 2].getDuration();
+					float leftF0 = waveTimeline.getSampleRate() / (float) leftDatagrams[0].getDuration();
+					float rightF0 = waveTimeline.getSampleRate()
+							/ (float) rightDatagrams[rightDatagrams.length - 1].getDuration();
+					System.out.println("Syllable at " + mid + " (length " + (last - first + 1) + "): left = " + ((int) leftF0)
+							+ ", mid = " + ((int) midF0) + ", right = " + rightF0);
+					toLeftFeaturesFile.println(leftF0 + " " + featureDefinition.toFeatureString(fvMid));
+					toMidFeaturesFile.println(midF0 + " " + featureDefinition.toFeatureString(fvMid));
+					toRightFeaturesFile.println(rightF0 + " " + featureDefinition.toFeatureString(fvMid));
+					nSyllables++;
+
+				}
+				// Skip the part we just covered:
+				i = last;
+			}
+		}
+		toLeftFeaturesFile.close();
+		toMidFeaturesFile.close();
+		toRightFeaturesFile.close();
+		System.out.println("F0 features extracted for " + nSyllables + " syllables");
+
+		if (useStepwiseTraining)
+			percent = 1; // estimated
+		else
+			percent = 10; // estimated
+		PrintWriter toDesc = new PrintWriter(new FileOutputStream(f0DescFile));
+		generateFeatureDescriptionForWagon(featureDefinition, toDesc);
+		toDesc.close();
+
+		// Now, call wagon
+		WagonCaller wagonCaller = new WagonCaller(db.getProp(DatabaseLayout.ESTDIR), null);
+		boolean ok;
+		if (useStepwiseTraining) {
+			// Split the data set in training and test part:
+			// TODO: hardcoded path = EVIL
+			Process traintest = Runtime.getRuntime().exec(
+					"/project/mary/Festival/festvox/src/general/traintest " + leftF0FeaturesFile.getAbsolutePath());
+			try {
+				traintest.waitFor();
+			} catch (InterruptedException ie) {
+			}
+			ok = wagonCaller.callWagon("-data " + leftF0FeaturesFile.getAbsolutePath() + ".train" + " -test "
+					+ leftF0FeaturesFile.getAbsolutePath() + ".test -stepwise" + " -desc " + f0DescFile.getAbsolutePath()
+					+ " -stop 10 " + " -output " + wagonLeftTreeFile.getAbsolutePath());
+		} else {
+			ok = wagonCaller.callWagon("-data " + leftF0FeaturesFile.getAbsolutePath() + " -desc " + f0DescFile.getAbsolutePath()
+					+ " -stop 10 " + " -output " + wagonLeftTreeFile.getAbsolutePath());
+		}
+		if (!ok)
+			return false;
+		percent = 40;
+		if (useStepwiseTraining) {
+			// Split the data set in training and test part:
+			// hardcoded path = EVIL
+			Process traintest = Runtime.getRuntime().exec(
+					"/project/mary/Festival/festvox/src/general/traintest " + midF0FeaturesFile.getAbsolutePath());
+			try {
+				traintest.waitFor();
+			} catch (InterruptedException ie) {
+			}
+			ok = wagonCaller.callWagon("-data " + midF0FeaturesFile.getAbsolutePath() + ".train" + " -test "
+					+ midF0FeaturesFile.getAbsolutePath() + ".test -stepwise" + " -desc " + f0DescFile.getAbsolutePath()
+					+ " -stop 10 " + " -output " + wagonMidTreeFile.getAbsolutePath());
+		} else {
+			ok = wagonCaller.callWagon("-data " + midF0FeaturesFile.getAbsolutePath() + " -desc " + f0DescFile.getAbsolutePath()
+					+ " -stop 10 " + " -output " + wagonMidTreeFile.getAbsolutePath());
+		}
+		if (!ok)
+			return false;
+		percent = 70;
+		if (useStepwiseTraining) {
+			// Split the data set in training and test part:
+			// hardcoded path = EVIL
+			Process traintest = Runtime.getRuntime().exec(
+					"/project/mary/Festival/festvox/src/general/traintest " + rightF0FeaturesFile.getAbsolutePath());
+			try {
+				traintest.waitFor();
+			} catch (InterruptedException ie) {
+			}
+			ok = wagonCaller.callWagon("-data " + rightF0FeaturesFile.getAbsolutePath() + ".train" + " -test "
+					+ rightF0FeaturesFile.getAbsolutePath() + ".test -stepwise" + " -desc " + f0DescFile.getAbsolutePath()
+					+ " -stop 10 " + " -output " + wagonRightTreeFile.getAbsolutePath());
+		} else {
+			ok = wagonCaller.callWagon("-data " + rightF0FeaturesFile.getAbsolutePath() + " -desc "
+					+ f0DescFile.getAbsolutePath() + " -stop 10 " + " -output " + wagonRightTreeFile.getAbsolutePath());
+		}
+
+		if (ok) {
+
+			// F0 Left file
+			String destinationFile = getProp(F0LEFTTREEFILE);
+			WagonCARTReader wagonLReader = new WagonCARTReader(LeafType.FloatLeafNode);
+			marytts.cart.Node rootLNode = wagonLReader.load(new BufferedReader(new FileReader(wagonLeftTreeFile)),
+					featureDefinition);
+			CART leftF0Cart = new CART(rootLNode, featureDefinition);
+			MaryCARTWriter wwl = new MaryCARTWriter();
+			wwl.dumpMaryCART(leftF0Cart, destinationFile);
+
+			// F0 Mid tree
+			destinationFile = getProp(F0MIDTREEFILE);
+			WagonCARTReader wagonMReader = new WagonCARTReader(LeafType.FloatLeafNode);
+			marytts.cart.Node rootMNode = wagonMReader.load(new BufferedReader(new FileReader(wagonMidTreeFile)),
+					featureDefinition);
+			CART midF0Cart = new CART(rootMNode, featureDefinition);
+			MaryCARTWriter wwm = new MaryCARTWriter();
+			wwm.dumpMaryCART(midF0Cart, destinationFile);
+
+			// F0 Right tree
+			destinationFile = getProp(F0RIGHTTREEFILE);
+			WagonCARTReader wagonRReader = new WagonCARTReader(LeafType.FloatLeafNode);
+			marytts.cart.Node rootRNode = wagonRReader.load(new BufferedReader(new FileReader(wagonRightTreeFile)),
+					featureDefinition);
+			CART rightF0Cart = new CART(rootRNode, featureDefinition);
+			MaryCARTWriter wwr = new MaryCARTWriter();
+			wwr.dumpMaryCART(rightF0Cart, destinationFile);
+
+		}
+
+		percent = 100;
+
+		return ok;
+	}
+
+	private String[] align(String basename) throws IOException {
+		BufferedReader labels = new BufferedReader(new InputStreamReader(new FileInputStream(new File(
+				db.getProp(DatabaseLayout.PHONELABDIR), basename + DatabaseLayout.LABEXT)), "UTF-8"));
+		BufferedReader features = new BufferedReader(new InputStreamReader(new FileInputStream(new File(
+				db.getProp(DatabaseLayout.PHONEFEATUREDIR), basename + featureExt)), "UTF-8"));
+		String line;
+		// Skip label file header:
+		while ((line = labels.readLine()) != null) {
+			if (line.startsWith("#"))
+				break; // line starting with "#" marks end of header
+		}
+		// Skip features file header:
+		while ((line = features.readLine()) != null) {
+			if (line.trim().equals(""))
+				break; // empty line marks end of header
+		}
+
+		// Now go through all feature file units
+		boolean correct = true;
+		int unitIndex = -1;
+		float prevEnd = 0;
+		List<String> aligned = new ArrayList<String>();
+		while (correct) {
+			unitIndex++;
+			String labelLine = labels.readLine();
+			String featureLine = features.readLine();
+			if (featureLine == null) { // incomplete feature file
+				return null;
+			} else if (featureLine.trim().equals("")) { // end of features
+				if (labelLine == null)
+					break; // normal end found
+				else
+					// label file is longer than feature file
+					return null;
+			}
+			// Verify that the two labels are the same:
+			StringTokenizer st = new StringTokenizer(labelLine.trim());
+			// The third token in each line is the label
+			float end = Float.parseFloat(st.nextToken());
+			st.nextToken(); // skip
+			String labelUnit = st.nextToken();
+
+			st = new StringTokenizer(featureLine.trim());
+			// The expect that the first token in each line is the label
+			String featureUnit = st.nextToken();
+			if (!featureUnit.equals(labelUnit)) {
+				// Non-matching units found
+				return null;
+			}
+			// OK, now we assume we have two matching lines.
+			if (!featureUnit.startsWith("_")) { // discard all silences
+				// Output format: unit duration, followed by the feature line
+				aligned.add(new PrintfFormat(Locale.ENGLISH, "%.3f").sprintf(end - prevEnd) + " " + featureLine.trim());
+			}
+			prevEnd = end;
+		}
+		return (String[]) aligned.toArray(new String[0]);
+	}
+
+	private void generateFeatureDescriptionForWagon(FeatureDefinition fd, PrintWriter out) {
+		out.println("(");
+		out.println("(f0 float)");
+		int nDiscreteFeatures = fd.getNumberOfByteFeatures() + fd.getNumberOfShortFeatures();
+		for (int i = 0, n = fd.getNumberOfFeatures(); i < n; i++) {
+			out.print("( ");
+			out.print(fd.getFeatureName(i));
+			if (i < nDiscreteFeatures) { // list values
+				if (fd.getNumberOfValues(i) == 20 && fd.getFeatureValueAsString(i, 19).equals("19")) {
+					// one of our pseudo-floats
+					out.println(" float )");
+				} else { // list the values
+					for (int v = 0, vmax = fd.getNumberOfValues(i); v < vmax; v++) {
+						out.print("  ");
+						String val = fd.getFeatureValueAsString(i, v);
+						if (val.indexOf('"') != -1) {
+							StringBuilder buf = new StringBuilder();
+							for (int c = 0; c < val.length(); c++) {
+								char ch = val.charAt(c);
+								if (ch == '"')
+									buf.append("\\\"");
+								else
+									buf.append(ch);
+							}
+							val = buf.toString();
+						}
+						out.print("\"" + val + "\"");
+					}
+					out.println(" )");
+				}
+			} else { // float feature
+				out.println(" float )");
+			}
+
+		}
+		out.println(")");
+	}
+
+	/**
+	 * Provide the progress of computation, in percent, or -1 if that feature is not implemented.
+	 * 
+	 * @return -1 if not implemented, or an integer between 0 and 100.
+	 */
+	public int getProgress() {
+		return percent;
+	}
+
+	public static void main(String[] args) throws Exception {
+		F0CARTTrainer f0ct = new F0CARTTrainer();
+		DatabaseLayout db = new DatabaseLayout(f0ct);
+		f0ct.compute();
+	}
 
 }
-

marytts-builder/src/main/java/marytts/tools/voiceimport/F0PolynomialInspector.java

@@ -53,350 +53,335 @@ import marytts.util.math.ArrayUtils;
 import marytts.util.math.Polynomial;
 import marytts.util.signal.SignalProcUtils;
 
+public class F0PolynomialInspector extends VoiceImportComponent {
+	protected FeatureFileReader features;
+	protected FeatureDefinition inFeatureDefinition;
+	protected UnitFileReader units;
+	protected FeatureFileReader contours;
+	protected TimelineReader audio;
+	protected DatabaseLayout db = null;
+	protected int percent = 0;
 
-public class F0PolynomialInspector extends VoiceImportComponent
-{
-    protected FeatureFileReader features;
-    protected FeatureDefinition inFeatureDefinition;
-    protected UnitFileReader units;
-    protected FeatureFileReader contours;
-    protected TimelineReader audio;
-    protected DatabaseLayout db = null;
-    protected int percent = 0;
-    
-    private final String name = "F0PolynomialInspector";
-    public final String UNITFILE = name + ".unitFile";
-    public final String WAVETIMELINE = name + ".waveTimeLine";
-    public final String ISHNMTIMELINE = name+".isHnmTimeline";
-    public final String FEATUREFILE = name + ".featureFile";
-    public final String F0FEATUREFILE = name + ".f0FeatureFile";  
-    
-    public String getName(){
-        return name;
-    }
-   
-    
-   public SortedMap<String, String> getDefaultProps(DatabaseLayout db){
-       this.db = db;
-       if (props == null){
-           props = new TreeMap<String, String>();
-           String fileDir = db.getProp(db.FILEDIR);
-           String maryExt = db.getProp(db.MARYEXT);
-           props.put(UNITFILE,fileDir+"halfphoneUnits"+maryExt);
-           props.put(WAVETIMELINE,fileDir+"timeline_waveforms"+maryExt);
-           props.put(ISHNMTIMELINE, "false"); 
-           props.put(FEATUREFILE,fileDir+"halfphoneFeatures_ac"+maryExt);
-           props.put(F0FEATUREFILE,fileDir+"syllableF0Polynomials"+maryExt);
-       }
-       return props;
-   }
-    
-   protected void setupHelp()
-   {
-       if (props2Help ==null) {
-           props2Help = new TreeMap<String, String>();
-           props2Help.put(UNITFILE,"file containing all halfphone units");
-           props2Help.put(WAVETIMELINE, "file containing all waveforms or models that can genarate them"); 
-           props2Help.put(ISHNMTIMELINE, "file containing all wave files");  
-           props2Help.put(FEATUREFILE,"file containing all halfphone units and their target cost features");
-           props2Help.put(F0FEATUREFILE,"file containing syllable-based polynom coefficients on vowels");
-       }
-   }
+	private final String name = "F0PolynomialInspector";
+	public final String UNITFILE = name + ".unitFile";
+	public final String WAVETIMELINE = name + ".waveTimeLine";
+	public final String ISHNMTIMELINE = name + ".isHnmTimeline";
+	public final String FEATUREFILE = name + ".featureFile";
+	public final String F0FEATUREFILE = name + ".f0FeatureFile";
 
-    @Override
-    public boolean compute() throws IOException, MaryConfigurationException
-    {
-        logger.info("F0 polynomial feature file writer started.");
+	public String getName() {
+		return name;
+	}
 
-        units = new UnitFileReader(getProp(UNITFILE));
-        audio = null;
-        if (getProp(ISHNMTIMELINE).compareToIgnoreCase("true")==0)
-            audio = new HnmTimelineReader(getProp(WAVETIMELINE));
-        else
-            audio = new TimelineReader(getProp(WAVETIMELINE));
-        
-        features = new FeatureFileReader(getProp(FEATUREFILE));
-        inFeatureDefinition = features.getFeatureDefinition();
-        StringWriter sw = new StringWriter();
-        PrintWriter pw = new PrintWriter(sw);
-        pw.println(FeatureDefinition.BYTEFEATURES); // no byte features
-        pw.println(FeatureDefinition.SHORTFEATURES); // no short features
-        pw.println(FeatureDefinition.CONTINUOUSFEATURES);
+	public SortedMap<String, String> getDefaultProps(DatabaseLayout db) {
+		this.db = db;
+		if (props == null) {
+			props = new TreeMap<String, String>();
+			String fileDir = db.getProp(db.FILEDIR);
+			String maryExt = db.getProp(db.MARYEXT);
+			props.put(UNITFILE, fileDir + "halfphoneUnits" + maryExt);
+			props.put(WAVETIMELINE, fileDir + "timeline_waveforms" + maryExt);
+			props.put(ISHNMTIMELINE, "false");
+			props.put(FEATUREFILE, fileDir + "halfphoneFeatures_ac" + maryExt);
+			props.put(F0FEATUREFILE, fileDir + "syllableF0Polynomials" + maryExt);
+		}
+		return props;
+	}
 
-        contours = new FeatureFileReader(getProp(F0FEATUREFILE));
+	protected void setupHelp() {
+		if (props2Help == null) {
+			props2Help = new TreeMap<String, String>();
+			props2Help.put(UNITFILE, "file containing all halfphone units");
+			props2Help.put(WAVETIMELINE, "file containing all waveforms or models that can genarate them");
+			props2Help.put(ISHNMTIMELINE, "file containing all wave files");
+			props2Help.put(FEATUREFILE, "file containing all halfphone units and their target cost features");
+			props2Help.put(F0FEATUREFILE, "file containing syllable-based polynom coefficients on vowels");
+		}
+	}
 
-        displaySentences();
-        return true;
-    }
+	@Override
+	public boolean compute() throws IOException, MaryConfigurationException {
+		logger.info("F0 polynomial feature file writer started.");
 
+		units = new UnitFileReader(getProp(UNITFILE));
+		audio = null;
+		if (getProp(ISHNMTIMELINE).compareToIgnoreCase("true") == 0)
+			audio = new HnmTimelineReader(getProp(WAVETIMELINE));
+		else
+			audio = new TimelineReader(getProp(WAVETIMELINE));
 
+		features = new FeatureFileReader(getProp(FEATUREFILE));
+		inFeatureDefinition = features.getFeatureDefinition();
+		StringWriter sw = new StringWriter();
+		PrintWriter pw = new PrintWriter(sw);
+		pw.println(FeatureDefinition.BYTEFEATURES); // no byte features
+		pw.println(FeatureDefinition.SHORTFEATURES); // no short features
+		pw.println(FeatureDefinition.CONTINUOUSFEATURES);
 
-    /**
-     * @param out
-     * @throws IOException
-     * @throws UnsupportedEncodingException
-     * @throws FileNotFoundException
-     */
-    protected void displaySentences() throws IOException
-    {
-        int numUnits = units.getNumberOfUnits();
-        int unitSampleRate = units.getSampleRate();
-        int audioSampleRate = audio.getSampleRate();
-        int unitIndex = 0;
+		contours = new FeatureFileReader(getProp(F0FEATUREFILE));
 
-        logger.debug("Number of units : "+numUnits);
+		displaySentences();
+		return true;
+	}
 
-        FeatureDefinition featureDefinition = features.getFeatureDefinition();
-        int fiPhoneme = featureDefinition.getFeatureIndex("phone");
-        byte fvPhoneme_0 = featureDefinition.getFeatureValueAsByte(fiPhoneme, "0");
-        byte fvPhoneme_Silence = featureDefinition.getFeatureValueAsByte(fiPhoneme, "_");
-        int fiLR = featureDefinition.getFeatureIndex("halfphone_lr");
-        byte fvLR_L = featureDefinition.getFeatureValueAsByte(fiLR, "L");
-        byte fvLR_R = featureDefinition.getFeatureValueAsByte(fiLR, "R");
-        int fiSylStart = featureDefinition.getFeatureIndex("segs_from_syl_start");
-        int fiSylEnd = featureDefinition.getFeatureIndex("segs_from_syl_end");
-        int fiSentenceStart = featureDefinition.getFeatureIndex("words_from_sentence_start");
-        int fiSentenceEnd = featureDefinition.getFeatureIndex("words_from_sentence_end");
-        int fiWordStart = featureDefinition.getFeatureIndex("segs_from_word_start");
-        int fiWordEnd = featureDefinition.getFeatureIndex("segs_from_word_end");
-        int fiVowel = featureDefinition.getFeatureIndex("ph_vc");
-        byte fvVowel_Plus = featureDefinition.getFeatureValueAsByte(fiVowel, "+");
-        
-        boolean haveUnitLogF0 = false;
-        int fiUnitLogF0 = -1;
-        int fiUnitLogF0delta = -1;
-        if (featureDefinition.hasFeature("unit_logf0") && featureDefinition.hasFeature("unit_logf0delta")) {
-            haveUnitLogF0 = true;
-            fiUnitLogF0 = featureDefinition.getFeatureIndex("unit_logf0");
-            fiUnitLogF0delta = featureDefinition.getFeatureIndex("unit_logf0delta");
-        }
-        
-        FunctionGraph f0Graph = null;
-        JFrame jf = null;
-        int iSentenceStart = -1;
-        int iSentenceEnd = -1;
-        List<Integer> iSylStarts = new ArrayList<Integer>(); 
-        List<Integer> iSylEnds = new ArrayList<Integer>();
-        List<Integer> iSylVowels = new ArrayList<Integer>();
-        f0Graph = new FunctionGraph(0, 1, new double[1]);
-        f0Graph.setYMinMax(50, 300);
-        f0Graph.setPrimaryDataSeriesStyle(Color.BLUE, FunctionGraph.DRAW_DOTS, FunctionGraph.DOT_FULLCIRCLE);
-        jf = f0Graph.showInJFrame("Sentence", false, true);
+	/**
+	 * @param out
+	 * @throws IOException
+	 * @throws UnsupportedEncodingException
+	 * @throws FileNotFoundException
+	 */
+	protected void displaySentences() throws IOException {
+		int numUnits = units.getNumberOfUnits();
+		int unitSampleRate = units.getSampleRate();
+		int audioSampleRate = audio.getSampleRate();
+		int unitIndex = 0;
 
-        for (int i=0; i<numUnits; i++) {
-            percent = 100*i/numUnits;
-            FeatureVector fv = features.getFeatureVector(i);
-            //System.out.print(featureDefinition.getFeatureValueAsString("phone", fv));
-            //if (fv.getByteFeature(fiPhoneme) == fvPhoneme_0
-            //        || fv.getByteFeature(fiPhoneme) == fvPhoneme_Silence) continue;
-            if (iSentenceStart == -1
-                    && fv.getByteFeature(fiSentenceStart) == 0
-                    && fv.getByteFeature(fiWordStart) == 0
-                    && fv.getByteFeature(fiLR) == fvLR_L) { // first unit in sentence
-                iSentenceStart = i;
-                iSylStarts.clear();
-                iSylEnds.clear();
-                iSylVowels.clear();
-                //System.out.print(", is sentence start");
-            }
-            // Silence and edge units cannot be part of syllables, but they can 
-            // mark start/end of sentence:
-            if (fv.getByteFeature(fiPhoneme) != fvPhoneme_0
-                    && fv.getByteFeature(fiPhoneme) != fvPhoneme_Silence) {
-                if (fv.getByteFeature(fiSylStart) == 0 && fv.getByteFeature(fiLR) == fvLR_L) { // first segment in syllable
-                    if (iSylStarts.size() > iSylEnds.size()) {
-                        System.err.println("Syllable ends before other syllable starts!");
-                    }
-                    iSylStarts.add(i);
-                    //System.out.print(", is syl start");
-                }
-                if (fv.getByteFeature(fiVowel) == fvVowel_Plus && iSylVowels.size() < iSylStarts.size()) { // first vowel unit in syllable
-                    iSylVowels.add(i);
-                    //System.out.print(", is vowel");
-                }
-                if (fv.getByteFeature(fiSylEnd) == 0 && fv.getByteFeature(fiLR) == fvLR_R) { // last segment in syllable
-                    iSylEnds.add(i);
-                    //System.out.print(", is syl end");
-                    assert iSylStarts.size() == iSylEnds.size();
-                    if (iSylVowels.size() < iSylEnds.size()) {
-                        //System.err.println("Syllable contains no vowel -- skipping");
-                        iSylStarts.remove(iSylStarts.size() - 1);
-                        iSylEnds.remove(iSylEnds.size() - 1);
-                    }
-                }
-            }
-            if (iSentenceStart != -1
-                    && fv.getByteFeature(fiSentenceEnd) == 0
-                    && fv.getByteFeature(fiWordEnd) == 0
-                    && fv.getByteFeature(fiLR) == fvLR_R) { // last unit in sentence
-                iSentenceEnd = i;
-                //System.out.print(", is sentence end");
-                if (iSylEnds.size() < iSylStarts.size()) {
-                    System.err.println("Last syllable in sentence is not properly closed");
-                    iSylEnds.add(i);
-                }
-            }
-            //System.out.println();
-            
-            if (iSentenceStart >= 0 && iSentenceEnd >= iSentenceStart && iSylVowels.size() > 0) {
-                assert iSylStarts.size() == iSylEnds.size() : "Have "+iSylStarts.size()+" syllable starts, but "+iSylEnds.size()+" syllable ends!";
-                assert iSylStarts.size() == iSylVowels.size();
-                long tsSentenceStart = units.getUnit(iSentenceStart).startTime;
-                long tsSentenceEnd = units.getUnit(iSentenceEnd).startTime + units.getUnit(iSentenceEnd).duration;
-                long tsSentenceDuration = tsSentenceEnd - tsSentenceStart;
-                Datagram[] sentenceData = audio.getDatagrams(tsSentenceStart, tsSentenceDuration);
-                DatagramDoubleDataSource ddds = new DatagramDoubleDataSource(sentenceData);
-                double[] sentenceAudio = ddds.getAllData();
-                AudioPlayer ap = null;
-                ap = new AudioPlayer(new DDSAudioInputStream(new BufferedDoubleDataSource(sentenceAudio), new AudioFormat(AudioFormat.Encoding.PCM_SIGNED,
-                        audioSampleRate, // samples per second
-                        16, // bits per sample
-                        1, // mono
-                        2, // nr. of bytes per frame
-                        audioSampleRate, // nr. of frames per second
-                        true))); // big-endian;))
-                ap.start();
-                PitchFileHeader params = new PitchFileHeader();
-                params.fs = audioSampleRate;
-                F0TrackerAutocorrelationHeuristic tracker = new F0TrackerAutocorrelationHeuristic(params);
-                tracker.pitchAnalyze(new BufferedDoubleDataSource(sentenceAudio));
-                double frameShiftTime = tracker.getSkipSizeInSeconds();
-                double[] f0Array = tracker.getF0Contour();
-                if (f0Array != null) {
-                    for (int j=0; j<f0Array.length; j++) {
-                        if (f0Array[j] == 0) {
-                            f0Array[j] = Double.NaN;
-                        }
-                    }
-                    if (f0Array.length >= 3) {
-                        f0Array = SignalProcUtils.medianFilter(f0Array, 5);
-                    }
-                    f0Graph.updateData(0, tsSentenceDuration / (double) audioSampleRate /f0Array.length, f0Array);
-                    jf.repaint();
+		logger.debug("Number of units : " + numUnits);
 
-                    double[] f0AndInterpol;
-                    double[] interpol = new double[f0Array.length];
-                    Arrays.fill(interpol, Double.NaN);
-                    f0AndInterpol = new double[f0Array.length];
-                    int iLastValid = -1;
-                    for (int j=0; j<f0Array.length; j++) {
-                        if (!Double.isNaN(f0Array[j])) { // a valid value
-                            if (iLastValid == j-1) {
-                                // no need to interpolate
-                                f0AndInterpol[j] = f0Array[j];
-                            } else {
-                                // need to interpolate
-                                double prevF0;
-                                if (iLastValid < 0) { // we don't have a previous value -- use current one
-                                    prevF0 = f0Array[j];
-                                } else {
-                                    prevF0 = f0Array[iLastValid];
-                                }
-                                double delta = (f0Array[j]-prevF0)/(j-iLastValid);
-                                double f0 = prevF0;
-                                for (int k = iLastValid+1; k<j; k++) {
-                                    f0 += delta;
-                                    interpol[k] = f0;
-                                    f0AndInterpol[k] = f0;
-                                }
-                            }
-                            iLastValid = j;
-                        }
-                    }
-                    f0Graph.addDataSeries(interpol, Color.GREEN, FunctionGraph.DRAW_DOTS, FunctionGraph.DOT_EMPTYCIRCLE);
-                    jf.repaint();
+		FeatureDefinition featureDefinition = features.getFeatureDefinition();
+		int fiPhoneme = featureDefinition.getFeatureIndex("phone");
+		byte fvPhoneme_0 = featureDefinition.getFeatureValueAsByte(fiPhoneme, "0");
+		byte fvPhoneme_Silence = featureDefinition.getFeatureValueAsByte(fiPhoneme, "_");
+		int fiLR = featureDefinition.getFeatureIndex("halfphone_lr");
+		byte fvLR_L = featureDefinition.getFeatureValueAsByte(fiLR, "L");
+		byte fvLR_R = featureDefinition.getFeatureValueAsByte(fiLR, "R");
+		int fiSylStart = featureDefinition.getFeatureIndex("segs_from_syl_start");
+		int fiSylEnd = featureDefinition.getFeatureIndex("segs_from_syl_end");
+		int fiSentenceStart = featureDefinition.getFeatureIndex("words_from_sentence_start");
+		int fiSentenceEnd = featureDefinition.getFeatureIndex("words_from_sentence_end");
+		int fiWordStart = featureDefinition.getFeatureIndex("segs_from_word_start");
+		int fiWordEnd = featureDefinition.getFeatureIndex("segs_from_word_end");
+		int fiVowel = featureDefinition.getFeatureIndex("ph_vc");
+		byte fvVowel_Plus = featureDefinition.getFeatureValueAsByte(fiVowel, "+");
 
-                    double[] approx = new double[f0Array.length];
-                    Arrays.fill(approx, Double.NaN);
-                    for (int s=0; s<iSylStarts.size(); s++) {
-                        long tsSylStart = units.getUnit(iSylStarts.get(s)).startTime;
-                        long tsSylEnd = units.getUnit(iSylEnds.get(s)).startTime + units.getUnit(iSylEnds.get(s)).duration;
-                        long tsSylDuration = tsSylEnd - tsSylStart;
-                        int iSylVowel = iSylVowels.get(s);
-                        // now map time to position in f0AndInterpol array:
-                        int iSylStart = (int) (((double)(tsSylStart-tsSentenceStart)/tsSentenceDuration)*f0AndInterpol.length);
-                        assert iSylStart >= 0;
-                        int iSylEnd = iSylStart + (int) ((double) tsSylDuration / tsSentenceDuration * f0AndInterpol.length) + 1;
-                        if (iSylEnd > approx.length) iSylEnd = approx.length;
-                        //System.out.println("Syl "+s+" from "+iSylStart+" to "+iSylEnd+" out of "+f0AndInterpol.length);
-                        double[] sylF0 = new double[iSylEnd-iSylStart];
-                        float[] coeffs = contours.getFeatureVector(iSylVowel).getContinuousFeatures();
-                        double[] sylPred = Polynomial.generatePolynomialValues(ArrayUtils.copyFloat2Double(coeffs), sylF0.length, 0, 1);
-                        System.arraycopy(sylPred, 0, approx, iSylStart, sylPred.length);
-                    }
-                    for (int j=0; j<approx.length; j++) {
-                        approx[j] = Math.exp(approx[j]);
-                    }
-                    f0Graph.addDataSeries(approx, Color.RED, FunctionGraph.DRAW_LINE, -1);
-                    System.out.println();
+		boolean haveUnitLogF0 = false;
+		int fiUnitLogF0 = -1;
+		int fiUnitLogF0delta = -1;
+		if (featureDefinition.hasFeature("unit_logf0") && featureDefinition.hasFeature("unit_logf0delta")) {
+			haveUnitLogF0 = true;
+			fiUnitLogF0 = featureDefinition.getFeatureIndex("unit_logf0");
+			fiUnitLogF0delta = featureDefinition.getFeatureIndex("unit_logf0delta");
+		}
 
-                    
-                    if (haveUnitLogF0) {
-                        double[] unitF0 = new double[f0Array.length];
-                        Arrays.fill(unitF0, Double.NaN);
-                        for (int u=0; u+iSentenceStart<=iSentenceEnd; u++) {
-                            FeatureVector localFV = features.getFeatureVector(u+iSentenceStart);
-                            long tsUnitStart = units.getUnit(u+iSentenceStart).startTime;
-                            long tsUnitDuration = units.getUnit(u+iSentenceStart).duration;
-                            int iUnitStartInArray = (int) (unitF0.length * (tsUnitStart-tsSentenceStart) / tsSentenceDuration);
-                            int iUnitDurationInArray = (int)(unitF0.length * tsUnitDuration / tsSentenceDuration);
-                            //while (iUnitDurationInArray+iUnitStartInArray>unitF0.length) iUnitDurationInArray--;
-                            if (iUnitDurationInArray > 0) {
-                                float logF0 = localFV.getContinuousFeature(fiUnitLogF0);
-                                float logF0delta = localFV.getContinuousFeature(fiUnitLogF0delta);
-                                double[] coeffs = new double[2];
-                                // logF0 is value at 0.5, logF0delta is slope
-                                // coeffs[0] is slope, coeffs[1] is value at 0  => coeffs[1] + 0.5*slope = logF0
-                                coeffs[0] = logF0delta;
-                                coeffs[1] = logF0 - 0.5*logF0delta;
-                                double[] pred = Polynomial.generatePolynomialValues(coeffs, iUnitDurationInArray, 0, 1);
-                                System.arraycopy(pred, 0, unitF0, iUnitStartInArray, iUnitDurationInArray);
-                                iUnitStartInArray += iUnitDurationInArray;
-                            }
-                        }
-                        for (int j=0; j<unitF0.length; j++) {
-                            unitF0[j] = Math.exp(unitF0[j]);
-                        }
-                        f0Graph.addDataSeries(unitF0, Color.BLACK, FunctionGraph.DRAW_LINE, -1);
+		FunctionGraph f0Graph = null;
+		JFrame jf = null;
+		int iSentenceStart = -1;
+		int iSentenceEnd = -1;
+		List<Integer> iSylStarts = new ArrayList<Integer>();
+		List<Integer> iSylEnds = new ArrayList<Integer>();
+		List<Integer> iSylVowels = new ArrayList<Integer>();
+		f0Graph = new FunctionGraph(0, 1, new double[1]);
+		f0Graph.setYMinMax(50, 300);
+		f0Graph.setPrimaryDataSeriesStyle(Color.BLUE, FunctionGraph.DRAW_DOTS, FunctionGraph.DOT_FULLCIRCLE);
+		jf = f0Graph.showInJFrame("Sentence", false, true);
 
-                    }
-                }
-                try {
-                    ap.join();
-                    Thread.sleep(4000);
-                } catch (InterruptedException ie) {}
-                iSentenceStart = -1;
-                iSentenceEnd = -1;
-                iSylStarts.clear();
-                iSylEnds.clear();
-                iSylVowels.clear();
-            }
-        }
-    
-    }
+		for (int i = 0; i < numUnits; i++) {
+			percent = 100 * i / numUnits;
+			FeatureVector fv = features.getFeatureVector(i);
+			// System.out.print(featureDefinition.getFeatureValueAsString("phone", fv));
+			// if (fv.getByteFeature(fiPhoneme) == fvPhoneme_0
+			// || fv.getByteFeature(fiPhoneme) == fvPhoneme_Silence) continue;
+			if (iSentenceStart == -1 && fv.getByteFeature(fiSentenceStart) == 0 && fv.getByteFeature(fiWordStart) == 0
+					&& fv.getByteFeature(fiLR) == fvLR_L) { // first unit in sentence
+				iSentenceStart = i;
+				iSylStarts.clear();
+				iSylEnds.clear();
+				iSylVowels.clear();
+				// System.out.print(", is sentence start");
+			}
+			// Silence and edge units cannot be part of syllables, but they can
+			// mark start/end of sentence:
+			if (fv.getByteFeature(fiPhoneme) != fvPhoneme_0 && fv.getByteFeature(fiPhoneme) != fvPhoneme_Silence) {
+				if (fv.getByteFeature(fiSylStart) == 0 && fv.getByteFeature(fiLR) == fvLR_L) { // first segment in syllable
+					if (iSylStarts.size() > iSylEnds.size()) {
+						System.err.println("Syllable ends before other syllable starts!");
+					}
+					iSylStarts.add(i);
+					// System.out.print(", is syl start");
+				}
+				if (fv.getByteFeature(fiVowel) == fvVowel_Plus && iSylVowels.size() < iSylStarts.size()) { // first vowel unit in
+																											// syllable
+					iSylVowels.add(i);
+					// System.out.print(", is vowel");
+				}
+				if (fv.getByteFeature(fiSylEnd) == 0 && fv.getByteFeature(fiLR) == fvLR_R) { // last segment in syllable
+					iSylEnds.add(i);
+					// System.out.print(", is syl end");
+					assert iSylStarts.size() == iSylEnds.size();
+					if (iSylVowels.size() < iSylEnds.size()) {
+						// System.err.println("Syllable contains no vowel -- skipping");
+						iSylStarts.remove(iSylStarts.size() - 1);
+						iSylEnds.remove(iSylEnds.size() - 1);
+					}
+				}
+			}
+			if (iSentenceStart != -1 && fv.getByteFeature(fiSentenceEnd) == 0 && fv.getByteFeature(fiWordEnd) == 0
+					&& fv.getByteFeature(fiLR) == fvLR_R) { // last unit in sentence
+				iSentenceEnd = i;
+				// System.out.print(", is sentence end");
+				if (iSylEnds.size() < iSylStarts.size()) {
+					System.err.println("Last syllable in sentence is not properly closed");
+					iSylEnds.add(i);
+				}
+			}
+			// System.out.println();
 
+			if (iSentenceStart >= 0 && iSentenceEnd >= iSentenceStart && iSylVowels.size() > 0) {
+				assert iSylStarts.size() == iSylEnds.size() : "Have " + iSylStarts.size() + " syllable starts, but "
+						+ iSylEnds.size() + " syllable ends!";
+				assert iSylStarts.size() == iSylVowels.size();
+				long tsSentenceStart = units.getUnit(iSentenceStart).startTime;
+				long tsSentenceEnd = units.getUnit(iSentenceEnd).startTime + units.getUnit(iSentenceEnd).duration;
+				long tsSentenceDuration = tsSentenceEnd - tsSentenceStart;
+				Datagram[] sentenceData = audio.getDatagrams(tsSentenceStart, tsSentenceDuration);
+				DatagramDoubleDataSource ddds = new DatagramDoubleDataSource(sentenceData);
+				double[] sentenceAudio = ddds.getAllData();
+				AudioPlayer ap = null;
+				ap = new AudioPlayer(new DDSAudioInputStream(new BufferedDoubleDataSource(sentenceAudio), new AudioFormat(
+						AudioFormat.Encoding.PCM_SIGNED, audioSampleRate, // samples per second
+						16, // bits per sample
+						1, // mono
+						2, // nr. of bytes per frame
+						audioSampleRate, // nr. of frames per second
+						true))); // big-endian;))
+				ap.start();
+				PitchFileHeader params = new PitchFileHeader();
+				params.fs = audioSampleRate;
+				F0TrackerAutocorrelationHeuristic tracker = new F0TrackerAutocorrelationHeuristic(params);
+				tracker.pitchAnalyze(new BufferedDoubleDataSource(sentenceAudio));
+				double frameShiftTime = tracker.getSkipSizeInSeconds();
+				double[] f0Array = tracker.getF0Contour();
+				if (f0Array != null) {
+					for (int j = 0; j < f0Array.length; j++) {
+						if (f0Array[j] == 0) {
+							f0Array[j] = Double.NaN;
+						}
+					}
+					if (f0Array.length >= 3) {
+						f0Array = SignalProcUtils.medianFilter(f0Array, 5);
+					}
+					f0Graph.updateData(0, tsSentenceDuration / (double) audioSampleRate / f0Array.length, f0Array);
+					jf.repaint();
 
-    /**
-     * Provide the progress of computation, in percent, or -1 if
-     * that feature is not implemented.
-     * @return -1 if not implemented, or an integer between 0 and 100.
-     */
-    public int getProgress()
-    {
-        return percent;
-    }
+					double[] f0AndInterpol;
+					double[] interpol = new double[f0Array.length];
+					Arrays.fill(interpol, Double.NaN);
+					f0AndInterpol = new double[f0Array.length];
+					int iLastValid = -1;
+					for (int j = 0; j < f0Array.length; j++) {
+						if (!Double.isNaN(f0Array[j])) { // a valid value
+							if (iLastValid == j - 1) {
+								// no need to interpolate
+								f0AndInterpol[j] = f0Array[j];
+							} else {
+								// need to interpolate
+								double prevF0;
+								if (iLastValid < 0) { // we don't have a previous value -- use current one
+									prevF0 = f0Array[j];
+								} else {
+									prevF0 = f0Array[iLastValid];
+								}
+								double delta = (f0Array[j] - prevF0) / (j - iLastValid);
+								double f0 = prevF0;
+								for (int k = iLastValid + 1; k < j; k++) {
+									f0 += delta;
+									interpol[k] = f0;
+									f0AndInterpol[k] = f0;
+								}
+							}
+							iLastValid = j;
+						}
+					}
+					f0Graph.addDataSeries(interpol, Color.GREEN, FunctionGraph.DRAW_DOTS, FunctionGraph.DOT_EMPTYCIRCLE);
+					jf.repaint();
 
-    /**
-     * @param args
-     */
-    public static void main(String[] args) throws Exception
-    {
-        F0PolynomialInspector acfeatsWriter = 
-            new F0PolynomialInspector();
-        DatabaseLayout db = new DatabaseLayout(acfeatsWriter);
-        acfeatsWriter.compute();
-    }
+					double[] approx = new double[f0Array.length];
+					Arrays.fill(approx, Double.NaN);
+					for (int s = 0; s < iSylStarts.size(); s++) {
+						long tsSylStart = units.getUnit(iSylStarts.get(s)).startTime;
+						long tsSylEnd = units.getUnit(iSylEnds.get(s)).startTime + units.getUnit(iSylEnds.get(s)).duration;
+						long tsSylDuration = tsSylEnd - tsSylStart;
+						int iSylVowel = iSylVowels.get(s);
+						// now map time to position in f0AndInterpol array:
+						int iSylStart = (int) (((double) (tsSylStart - tsSentenceStart) / tsSentenceDuration) * f0AndInterpol.length);
+						assert iSylStart >= 0;
+						int iSylEnd = iSylStart + (int) ((double) tsSylDuration / tsSentenceDuration * f0AndInterpol.length) + 1;
+						if (iSylEnd > approx.length)
+							iSylEnd = approx.length;
+						// System.out.println("Syl "+s+" from "+iSylStart+" to "+iSylEnd+" out of "+f0AndInterpol.length);
+						double[] sylF0 = new double[iSylEnd - iSylStart];
+						float[] coeffs = contours.getFeatureVector(iSylVowel).getContinuousFeatures();
+						double[] sylPred = Polynomial.generatePolynomialValues(ArrayUtils.copyFloat2Double(coeffs), sylF0.length,
+								0, 1);
+						System.arraycopy(sylPred, 0, approx, iSylStart, sylPred.length);
+					}
+					for (int j = 0; j < approx.length; j++) {
+						approx[j] = Math.exp(approx[j]);
+					}
+					f0Graph.addDataSeries(approx, Color.RED, FunctionGraph.DRAW_LINE, -1);
+					System.out.println();
 
+					if (haveUnitLogF0) {
+						double[] unitF0 = new double[f0Array.length];
+						Arrays.fill(unitF0, Double.NaN);
+						for (int u = 0; u + iSentenceStart <= iSentenceEnd; u++) {
+							FeatureVector localFV = features.getFeatureVector(u + iSentenceStart);
+							long tsUnitStart = units.getUnit(u + iSentenceStart).startTime;
+							long tsUnitDuration = units.getUnit(u + iSentenceStart).duration;
+							int iUnitStartInArray = (int) (unitF0.length * (tsUnitStart - tsSentenceStart) / tsSentenceDuration);
+							int iUnitDurationInArray = (int) (unitF0.length * tsUnitDuration / tsSentenceDuration);
+							// while (iUnitDurationInArray+iUnitStartInArray>unitF0.length) iUnitDurationInArray--;
+							if (iUnitDurationInArray > 0) {
+								float logF0 = localFV.getContinuousFeature(fiUnitLogF0);
+								float logF0delta = localFV.getContinuousFeature(fiUnitLogF0delta);
+								double[] coeffs = new double[2];
+								// logF0 is value at 0.5, logF0delta is slope
+								// coeffs[0] is slope, coeffs[1] is value at 0 => coeffs[1] + 0.5*slope = logF0
+								coeffs[0] = logF0delta;
+								coeffs[1] = logF0 - 0.5 * logF0delta;
+								double[] pred = Polynomial.generatePolynomialValues(coeffs, iUnitDurationInArray, 0, 1);
+								System.arraycopy(pred, 0, unitF0, iUnitStartInArray, iUnitDurationInArray);
+								iUnitStartInArray += iUnitDurationInArray;
+							}
+						}
+						for (int j = 0; j < unitF0.length; j++) {
+							unitF0[j] = Math.exp(unitF0[j]);
+						}
+						f0Graph.addDataSeries(unitF0, Color.BLACK, FunctionGraph.DRAW_LINE, -1);
+
+					}
+				}
+				try {
+					ap.join();
+					Thread.sleep(4000);
+				} catch (InterruptedException ie) {
+				}
+				iSentenceStart = -1;
+				iSentenceEnd = -1;
+				iSylStarts.clear();
+				iSylEnds.clear();
+				iSylVowels.clear();
+			}
+		}
+
+	}
+
+	/**
+	 * Provide the progress of computation, in percent, or -1 if that feature is not implemented.
+	 * 
+	 * @return -1 if not implemented, or an integer between 0 and 100.
+	 */
+	public int getProgress() {
+		return percent;
+	}
+
+	/**
+	 * @param args
+	 */
+	public static void main(String[] args) throws Exception {
+		F0PolynomialInspector acfeatsWriter = new F0PolynomialInspector();
+		DatabaseLayout db = new DatabaseLayout(acfeatsWriter);
+		acfeatsWriter.compute();
+	}
 
 }
-

marytts-builder/src/main/java/marytts/tools/voiceimport/F0PolynomialTreeTrainer.java

@@ -38,189 +38,169 @@ import marytts.tools.voiceimport.traintrees.Wagon;
 import marytts.unitselection.data.FeatureFileReader;
 import marytts.util.math.ArrayUtils;
 
+public class F0PolynomialTreeTrainer extends VoiceImportComponent {
+	protected File maryDir;
+	protected FeatureFileReader features;
+	protected FeatureDefinition featureDefinition;
+	protected FeatureFileReader contours;
+	protected DatabaseLayout db = null;
 
-public class F0PolynomialTreeTrainer extends VoiceImportComponent
-{
-    protected File maryDir;
-    protected FeatureFileReader features;
-    protected FeatureDefinition featureDefinition;
-    protected FeatureFileReader contours;
-    protected DatabaseLayout db = null;
-    
-    public final String FEATUREFILE = "F0PolynomialTreeTrainer.featureFile";
-    public final String F0FEATUREFILE = "F0PolynomialTreeTrainer.f0FeatureFile";  
-    public final String F0TREE = "F0PolynomialTreeTrainer.treeFile";
-    public final String MAXDATA = "F0PolynomialTreeTrainer.maxData";
-    public final String PROPORTIONTESTDATA = "F0PolynomialTreeTrainer.propTestData";
-    
-    public final String WAGONDIR = "F0PolynomialTreeTrainer.wagonDir";
-    public final String WAGONEXECUTABLE = "F0PolynomialTreeTrainer.wagonExecutable";
-    public final String BALANCE = "F0PolynomialTreeTrainer.wagonBalance";
-    public final String STOP = "F0PolynomialTreeTrainer.wagonStop";
+	public final String FEATUREFILE = "F0PolynomialTreeTrainer.featureFile";
+	public final String F0FEATUREFILE = "F0PolynomialTreeTrainer.f0FeatureFile";
+	public final String F0TREE = "F0PolynomialTreeTrainer.treeFile";
+	public final String MAXDATA = "F0PolynomialTreeTrainer.maxData";
+	public final String PROPORTIONTESTDATA = "F0PolynomialTreeTrainer.propTestData";
 
-    public String getName() {
-        return "F0PolynomialTreeTrainer";
-    }
-   
-    
-   public SortedMap<String, String> getDefaultProps(DatabaseLayout theDb){
-       this.db = theDb;
-       if (props == null) {
-           props = new TreeMap<String, String>();
-           String fileDir = db.getProp(db.FILEDIR);
-           String maryExt = db.getProp(db.MARYEXT);
-           props.put(FEATUREFILE,fileDir+"halfphoneFeatures"+maryExt);
-           props.put(F0FEATUREFILE,fileDir+"syllableF0Polynomials"+maryExt);
-           props.put(F0TREE, fileDir+"f0contourtree.mry");
-           props.put(MAXDATA, "0");
-           props.put(PROPORTIONTESTDATA, "0.1");
-           
-           props.put(WAGONDIR, "f0contours");
-           props.put(WAGONEXECUTABLE, System.getenv("ESTDIR")+"/bin/wagon");
-           props.put(BALANCE, "0");
-           props.put(STOP, "50");
-       }
-       return props;
-   }
-    
-   protected void setupHelp()
-   {
-       if (props2Help ==null) {
-           props2Help = new TreeMap<String, String>();
-           props2Help.put(FEATUREFILE,"file containing all halfphone units and their target cost features");
-           props2Help.put(F0FEATUREFILE,"file containing syllable-based polynom coefficients on vowels");
-           props2Help.put(F0TREE, "the path for saving the resulting f0 contour tree");
-           props2Help.put(MAXDATA, "if >0, gives the maximum number of syllables to use for training the tree");
-           props2Help.put(PROPORTIONTESTDATA, "the proportion of the data to use as test data (choose so that 1/value is an integer)");
-           
-           props2Help.put(WAGONDIR, "directory in which to store the wagon files");
-           props2Help.put(WAGONEXECUTABLE, "full path of the wagon executable from the Edinburgh speech tools");
-           props2Help.put(BALANCE, "the wagon balance value");
-           props2Help.put(STOP, "the wagon stop criterion (min number of items in leaf)");
-       }
-   }
+	public final String WAGONDIR = "F0PolynomialTreeTrainer.wagonDir";
+	public final String WAGONEXECUTABLE = "F0PolynomialTreeTrainer.wagonExecutable";
+	public final String BALANCE = "F0PolynomialTreeTrainer.wagonBalance";
+	public final String STOP = "F0PolynomialTreeTrainer.wagonStop";
 
-    @Override
-    public boolean compute() throws IOException, MaryConfigurationException
-    {
-        logger.info("F0 polynomial tree trainer started.");
+	public String getName() {
+		return "F0PolynomialTreeTrainer";
+	}
 
-        features = new FeatureFileReader(getProp(FEATUREFILE));
-        featureDefinition = features.getFeatureDefinition();
-        contours = new FeatureFileReader(getProp(F0FEATUREFILE));
-        if (features.getNumberOfUnits() != contours.getNumberOfUnits()) {
-            throw new IllegalArgumentException("Number of units differs between feature file and contour file -- they are out of sync");
-        }
-        
-        int maxData = Integer.parseInt(getProp(MAXDATA));
-        
-        // 1. Extract the feature vectors for which there are contours
-        List<FeatureVector> relevantFVList = new ArrayList<FeatureVector>();
-        for (int i=0, numUnits=contours.getNumberOfUnits(); i<numUnits; i++) {
-            FeatureVector fv = contours.getFeatureVector(i);
-            float[] floats = fv.getContinuousFeatures();
-            boolean isZero = ArrayUtils.isZero(floats);
-            if (!isZero) {
-                relevantFVList.add(features.getFeatureVector(i));
-                if (maxData > 0 && relevantFVList.size() >= maxData) break;
-            }
-        }
-        FeatureVector[] relevantFV = relevantFVList.toArray(new FeatureVector[0]);
-        logger.debug("Read "+relevantFV.length+" f0 contours and corresponding feature vectors");
+	public SortedMap<String, String> getDefaultProps(DatabaseLayout theDb) {
+		this.db = theDb;
+		if (props == null) {
+			props = new TreeMap<String, String>();
+			String fileDir = db.getProp(db.FILEDIR);
+			String maryExt = db.getProp(db.MARYEXT);
+			props.put(FEATUREFILE, fileDir + "halfphoneFeatures" + maryExt);
+			props.put(F0FEATUREFILE, fileDir + "syllableF0Polynomials" + maryExt);
+			props.put(F0TREE, fileDir + "f0contourtree.mry");
+			props.put(MAXDATA, "0");
+			props.put(PROPORTIONTESTDATA, "0.1");
 
-        // 2. Grow a tree
-        //CART cart = trainWagon(relevantFV);
-        DirectedGraph graph = trainAgglomerativeCluster(relevantFV);
-        
-        // 3. Save resulting tree in MARY format.
-        if (graph != null) {
-            DirectedGraphWriter writer = new DirectedGraphWriter();
-            writer.saveGraph(graph, getProp(F0TREE));
-            return true;
-        }
-        return false;
-    }
+			props.put(WAGONDIR, "f0contours");
+			props.put(WAGONEXECUTABLE, System.getenv("ESTDIR") + "/bin/wagon");
+			props.put(BALANCE, "0");
+			props.put(STOP, "50");
+		}
+		return props;
+	}
 
+	protected void setupHelp() {
+		if (props2Help == null) {
+			props2Help = new TreeMap<String, String>();
+			props2Help.put(FEATUREFILE, "file containing all halfphone units and their target cost features");
+			props2Help.put(F0FEATUREFILE, "file containing syllable-based polynom coefficients on vowels");
+			props2Help.put(F0TREE, "the path for saving the resulting f0 contour tree");
+			props2Help.put(MAXDATA, "if >0, gives the maximum number of syllables to use for training the tree");
+			props2Help.put(PROPORTIONTESTDATA,
+					"the proportion of the data to use as test data (choose so that 1/value is an integer)");
 
-    private CART trainWagon(FeatureVector[] relevantFV) throws IOException
-    {
-        // 2. Call wagon with these feature vectors 
-        // and a distance measure based on contour distances
-        Wagon.setWagonExecutable(new File(getProp(WAGONEXECUTABLE)));
-        File wagonDir = new File(getProp(WAGONDIR));
-        if (!wagonDir.exists()) {
-            wagonDir.mkdirs();
-        }
-        Wagon w = new Wagon("f0contours", featureDefinition, relevantFV,
-                new F0ContourPolynomialDistanceMeasure(contours),
-                wagonDir,
-                Integer.parseInt(getProp(BALANCE)),
-                Integer.parseInt(getProp(STOP)));
-        w.run();
-        if (w.success()) return w.getCART();
-        return null;
-    }
+			props2Help.put(WAGONDIR, "directory in which to store the wagon files");
+			props2Help.put(WAGONEXECUTABLE, "full path of the wagon executable from the Edinburgh speech tools");
+			props2Help.put(BALANCE, "the wagon balance value");
+			props2Help.put(STOP, "the wagon stop criterion (min number of items in leaf)");
+		}
+	}
 
+	@Override
+	public boolean compute() throws IOException, MaryConfigurationException {
+		logger.info("F0 polynomial tree trainer started.");
 
-    private DirectedGraph trainAgglomerativeCluster(FeatureVector[] relevantFV)
-    throws IOException
-    {
-        List<String> featuresToUse = new ArrayList<String>();
-        for (int i=0, numByteFeatures = featureDefinition.getNumberOfByteFeatures(); i<numByteFeatures; i++) {
-            String f = featureDefinition.getFeatureName(i);
-            if (!f.contains("phone") && !f.contains("halfphone") &&
-                    !f.contains("vc") && !f.contains("ctype")
-                    && !f.contains("cvox") && !f.contains("edge")
-                    && !f.contains("vfront") && !f.contains("vlng")
-                    && !f.contains("vheight") && !f.contains("cplace")
-                    && !f.contains("vrnd") && !f.contains("selection_next_phone_class")) {
-                featuresToUse.add(f);
-                //System.out.println("adding feature "+f);
-            } else {
-                //System.err.println("ignoring feature "+f);
-            }
-        }
+		features = new FeatureFileReader(getProp(FEATUREFILE));
+		featureDefinition = features.getFeatureDefinition();
+		contours = new FeatureFileReader(getProp(F0FEATUREFILE));
+		if (features.getNumberOfUnits() != contours.getNumberOfUnits()) {
+			throw new IllegalArgumentException(
+					"Number of units differs between feature file and contour file -- they are out of sync");
+		}
 
-        AgglomerativeClusterer clusterer = new AgglomerativeClusterer(relevantFV,
-                featureDefinition,
-                featuresToUse,
-                new F0ContourPolynomialDistanceMeasure(contours));
-        DirectedGraphWriter writer = new DirectedGraphWriter();
-        DirectedGraph graph;
-        int iteration = 0;
-        do {
-            graph = clusterer.cluster();
-            iteration++;
-            if (graph != null) {
-                writer.saveGraph(graph, getProp(F0TREE)+".level"+iteration);
-            }            
-        } while (clusterer.canClusterMore());
-        return graph;
-    }
+		int maxData = Integer.parseInt(getProp(MAXDATA));
 
-    
-    
-    /**
-     * Provide the progress of computation, in percent, or -1 if
-     * that feature is not implemented.
-     * @return -1 if not implemented, or an integer between 0 and 100.
-     */
-    public int getProgress()
-    {
-        return -1;
-    }
+		// 1. Extract the feature vectors for which there are contours
+		List<FeatureVector> relevantFVList = new ArrayList<FeatureVector>();
+		for (int i = 0, numUnits = contours.getNumberOfUnits(); i < numUnits; i++) {
+			FeatureVector fv = contours.getFeatureVector(i);
+			float[] floats = fv.getContinuousFeatures();
+			boolean isZero = ArrayUtils.isZero(floats);
+			if (!isZero) {
+				relevantFVList.add(features.getFeatureVector(i));
+				if (maxData > 0 && relevantFVList.size() >= maxData)
+					break;
+			}
+		}
+		FeatureVector[] relevantFV = relevantFVList.toArray(new FeatureVector[0]);
+		logger.debug("Read " + relevantFV.length + " f0 contours and corresponding feature vectors");
 
-    /**
-     * @param args
-     */
-    public static void main(String[] args) throws Exception
-    {
-        F0PolynomialTreeTrainer acfeatsWriter = 
-            new F0PolynomialTreeTrainer();
-        DatabaseLayout db = new DatabaseLayout(acfeatsWriter);
-        acfeatsWriter.compute();
-    }
+		// 2. Grow a tree
+		// CART cart = trainWagon(relevantFV);
+		DirectedGraph graph = trainAgglomerativeCluster(relevantFV);
 
+		// 3. Save resulting tree in MARY format.
+		if (graph != null) {
+			DirectedGraphWriter writer = new DirectedGraphWriter();
+			writer.saveGraph(graph, getProp(F0TREE));
+			return true;
+		}
+		return false;
+	}
+
+	private CART trainWagon(FeatureVector[] relevantFV) throws IOException {
+		// 2. Call wagon with these feature vectors
+		// and a distance measure based on contour distances
+		Wagon.setWagonExecutable(new File(getProp(WAGONEXECUTABLE)));
+		File wagonDir = new File(getProp(WAGONDIR));
+		if (!wagonDir.exists()) {
+			wagonDir.mkdirs();
+		}
+		Wagon w = new Wagon("f0contours", featureDefinition, relevantFV, new F0ContourPolynomialDistanceMeasure(contours),
+				wagonDir, Integer.parseInt(getProp(BALANCE)), Integer.parseInt(getProp(STOP)));
+		w.run();
+		if (w.success())
+			return w.getCART();
+		return null;
+	}
+
+	private DirectedGraph trainAgglomerativeCluster(FeatureVector[] relevantFV) throws IOException {
+		List<String> featuresToUse = new ArrayList<String>();
+		for (int i = 0, numByteFeatures = featureDefinition.getNumberOfByteFeatures(); i < numByteFeatures; i++) {
+			String f = featureDefinition.getFeatureName(i);
+			if (!f.contains("phone") && !f.contains("halfphone") && !f.contains("vc") && !f.contains("ctype")
+					&& !f.contains("cvox") && !f.contains("edge") && !f.contains("vfront") && !f.contains("vlng")
+					&& !f.contains("vheight") && !f.contains("cplace") && !f.contains("vrnd")
+					&& !f.contains("selection_next_phone_class")) {
+				featuresToUse.add(f);
+				// System.out.println("adding feature "+f);
+			} else {
+				// System.err.println("ignoring feature "+f);
+			}
+		}
+
+		AgglomerativeClusterer clusterer = new AgglomerativeClusterer(relevantFV, featureDefinition, featuresToUse,
+				new F0ContourPolynomialDistanceMeasure(contours));
+		DirectedGraphWriter writer = new DirectedGraphWriter();
+		DirectedGraph graph;
+		int iteration = 0;
+		do {
+			graph = clusterer.cluster();
+			iteration++;
+			if (graph != null) {
+				writer.saveGraph(graph, getProp(F0TREE) + ".level" + iteration);
+			}
+		} while (clusterer.canClusterMore());
+		return graph;
+	}
+
+	/**
+	 * Provide the progress of computation, in percent, or -1 if that feature is not implemented.
+	 * 
+	 * @return -1 if not implemented, or an integer between 0 and 100.
+	 */
+	public int getProgress() {
+		return -1;
+	}
+
+	/**
+	 * @param args
+	 */
+	public static void main(String[] args) throws Exception {
+		F0PolynomialTreeTrainer acfeatsWriter = new F0PolynomialTreeTrainer();
+		DatabaseLayout db = new DatabaseLayout(acfeatsWriter);
+		acfeatsWriter.compute();
+	}
 
 }
-

marytts-builder/src/main/java/marytts/tools/voiceimport/F0SoPTrainer.java

@@ -47,439 +47,405 @@ import marytts.unitselection.data.TimelineReader;
 import marytts.unitselection.data.UnitFileReader;
 import marytts.util.data.Datagram;
 
-
-
 /**
  * 
  * Example of how to use SoP modules in a voice configuration file:
  * 
- * #Modules to use for predicting acoustic target features for this voice:
- * #this are the names that will be used to identify diferent models of the same type, sop, cart or hmm
- * voice.cmu-slt-hsmm.acousticModels = duration F0 midF0 rightF0
-
- * voice.cmu-slt-hsmm.duration.model = sop
- * voice.cmu-slt-hsmm.duration.data = MARY_BASE/lib/voices/cmu-slt-hsmm/dur.sop
+ * #Modules to use for predicting acoustic target features for this voice: #this are the names that will be used to identify
+ * diferent models of the same type, sop, cart or hmm voice.cmu-slt-hsmm.acousticModels = duration F0 midF0 rightF0
+ * 
+ * voice.cmu-slt-hsmm.duration.model = sop voice.cmu-slt-hsmm.duration.data = MARY_BASE/lib/voices/cmu-slt-hsmm/dur.sop
  * voice.cmu-slt-hsmm.duration.attribute = d
-
- * voice.cmu-slt-hsmm.leftF0.model = sop
- * voice.cmu-slt-hsmm.leftF0.data = MARY_BASE/lib/voices/cmu-slt-hsmm/f0.left.sop
- * voice.cmu-slt-hsmm.leftF0.attribute = f0
- * voice.cmu-slt-hsmm.leftF0.attribute.format = (0,%.0f)
-
- * voice.cmu-slt-hsmm.midF0.model = sop
- * voice.cmu-slt-hsmm.midF0.data = MARY_BASE/lib/voices/cmu-slt-hsmm/f0.mid.sop
- * voice.cmu-slt-hsmm.midF0.attribute = f0
- * voice.cmu-slt-hsmm.midF0.attribute.format = (50,%.0f)
-
- * voice.cmu-slt-hsmm.rightF0.model = sop
- * voice.cmu-slt-hsmm.rightF0.data = MARY_BASE/lib/voices/cmu-slt-hsmm/f0.right.sop
- * voice.cmu-slt-hsmm.rightF0.attribute = f0
- * voice.cmu-slt-hsmm.rightF0.attribute.format = (100,%.0f)
+ * 
+ * voice.cmu-slt-hsmm.leftF0.model = sop voice.cmu-slt-hsmm.leftF0.data = MARY_BASE/lib/voices/cmu-slt-hsmm/f0.left.sop
+ * voice.cmu-slt-hsmm.leftF0.attribute = f0 voice.cmu-slt-hsmm.leftF0.attribute.format = (0,%.0f)
+ * 
+ * voice.cmu-slt-hsmm.midF0.model = sop voice.cmu-slt-hsmm.midF0.data = MARY_BASE/lib/voices/cmu-slt-hsmm/f0.mid.sop
+ * voice.cmu-slt-hsmm.midF0.attribute = f0 voice.cmu-slt-hsmm.midF0.attribute.format = (50,%.0f)
+ * 
+ * voice.cmu-slt-hsmm.rightF0.model = sop voice.cmu-slt-hsmm.rightF0.data = MARY_BASE/lib/voices/cmu-slt-hsmm/f0.right.sop
+ * voice.cmu-slt-hsmm.rightF0.attribute = f0 voice.cmu-slt-hsmm.rightF0.attribute.format = (100,%.0f)
  * 
  * @author marcela
  *
  */
-public class F0SoPTrainer extends VoiceImportComponent
-{
-  protected File f0Dir;
-  protected String leftF0FeaturesFileName;
-  protected String midF0FeaturesFileName;
-  protected String rightF0FeaturesFileName;
-  protected File leftSoPFile;
-  protected File midSoPFile;
-  protected File rightSoPFile;
-  protected String featureExt = ".pfeats";  
-  protected String labelExt = ".lab";
-  protected DatabaseLayout db = null;
-  protected int percent = 0;
-  protected boolean success = true;
-  protected boolean interceptTerm = true;
-  protected boolean logF0 = false;
-  protected int solutionSize;
-  
-  private final String name = "F0SoPTrainer";
-  
-  public final String FEATUREFILE = name+".featureFile";
-  public final String UNITFILE = name+".unitFile";
-  public final String WAVETIMELINE = name+".waveTimeline";
-  public final String LABELDIR = name+".labelDir";
-  public final String FEATUREDIR = name+".featureDir";
-  public final String F0LeftSoPFILE = name+".f0LeftSoPFile";
-  public final String F0MidSoPFILE = name+".f0MidSoPFile";
-  public final String F0RightSoPFILE = name+".f0RightSoPFile";
-  private final String SOLUTIONSIZE = name+".solutionSize";
-  private final String INTERCEPTTERM = name+".interceptTerm";
-  private final String LOGF0SOLUTION = name+".logF0";
+public class F0SoPTrainer extends VoiceImportComponent {
+	protected File f0Dir;
+	protected String leftF0FeaturesFileName;
+	protected String midF0FeaturesFileName;
+	protected String rightF0FeaturesFileName;
+	protected File leftSoPFile;
+	protected File midSoPFile;
+	protected File rightSoPFile;
+	protected String featureExt = ".pfeats";
+	protected String labelExt = ".lab";
+	protected DatabaseLayout db = null;
+	protected int percent = 0;
+	protected boolean success = true;
+	protected boolean interceptTerm = true;
+	protected boolean logF0 = false;
+	protected int solutionSize;
 
+	private final String name = "F0SoPTrainer";
 
-  
-  public F0SoPTrainer(){
-      setupHelp();
-  }
-  
-  public String getName(){
-      return name;
-  }
+	public final String FEATUREFILE = name + ".featureFile";
+	public final String UNITFILE = name + ".unitFile";
+	public final String WAVETIMELINE = name + ".waveTimeline";
+	public final String LABELDIR = name + ".labelDir";
+	public final String FEATUREDIR = name + ".featureDir";
+	public final String F0LeftSoPFILE = name + ".f0LeftSoPFile";
+	public final String F0MidSoPFILE = name + ".f0MidSoPFile";
+	public final String F0RightSoPFILE = name + ".f0RightSoPFile";
+	private final String SOLUTIONSIZE = name + ".solutionSize";
+	private final String INTERCEPTTERM = name + ".interceptTerm";
+	private final String LOGF0SOLUTION = name + ".logF0";
 
-  @Override
-  protected void initialiseComp()
-  {       
-      String rootDir = db.getProp(db.ROOTDIR);
-      String f0DirName = db.getProp(db.TEMPDIR);
-      this.f0Dir = new File(f0DirName);
-      if (!f0Dir.exists()){
-          System.out.print("temp dir " + f0DirName + " does not exist; ");
-          if (!f0Dir.mkdir()){
-              throw new Error("Could not create F0DIR");
-          }
-          System.out.print("Created successfully.\n");
-      }  
-      this.leftF0FeaturesFileName = f0Dir + "f0.sop.left.feats";
-      this.midF0FeaturesFileName = f0Dir + "f0.sop.mid.feats";
-      this.rightF0FeaturesFileName = f0Dir + "f0.sop.right.feats";
-      this.interceptTerm =  Boolean.valueOf(getProp(INTERCEPTTERM)).booleanValue(); 
-      this.logF0 =  Boolean.valueOf(getProp(LOGF0SOLUTION)).booleanValue();
-      this.solutionSize = Integer.parseInt(getProp(SOLUTIONSIZE));
-     
-  }
-   
-   public SortedMap<String, String> getDefaultProps(DatabaseLayout dbl){
-      this.db = dbl;
-     if (props == null){
-         props = new TreeMap<String, String>();
-         String filedir = db.getProp(db.FILEDIR);
-         String maryext = db.getProp(db.MARYEXT);
-         props.put(FEATUREDIR, db.getProp(db.ROOTDIR) + "phonefeatures" + System.getProperty("file.separator"));
-         props.put(LABELDIR, db.getProp(db.ROOTDIR) + "phonelab" + System.getProperty("file.separator"));
-         props.put(FEATUREFILE, filedir + "phoneFeatures"+maryext);
-         props.put(UNITFILE, filedir + "phoneUnits"+maryext);
-         props.put(WAVETIMELINE, db.getProp(db.FILEDIR) + "timeline_waveforms"+db.getProp(db.MARYEXT));      
-         props.put(F0LeftSoPFILE,filedir + "f0.left.sop");
-         props.put(F0MidSoPFILE,filedir + "f0.mid.sop");
-         props.put(F0RightSoPFILE,filedir + "f0.right.sop");
-         props.put(INTERCEPTTERM, "true");
-         props.put(LOGF0SOLUTION, "false");
-         props.put(SOLUTIONSIZE, "5");
+	public F0SoPTrainer() {
+		setupHelp();
+	}
 
-     }
-     return props;
-   }
-   
-   protected void setupHelp(){         
-       props2Help = new TreeMap<String, String>();
-       props2Help.put(FEATUREDIR, "directory containing the phonefeatures");
-       props2Help.put(LABELDIR, "directory containing the phone label files");
-       props2Help.put(FEATUREFILE, "file containing all phone units and their target cost features");
-       props2Help.put(UNITFILE, "file containing all phone units");
-       props2Help.put(WAVETIMELINE, "file containing all waveforms or models that can genarate them");   
-       props2Help.put(F0LeftSoPFILE,"file containing the f0 left SoP model. Will be created by this module");
-       props2Help.put(F0MidSoPFILE,"file containing the f0 mid SoP model. Will be created by this module");
-       props2Help.put(F0RightSoPFILE,"file containing the f0 right SoP model. Will be created by this module");
-       props2Help.put(INTERCEPTTERM, "whether to include interceptTerm (b0) on the solution equation : b0 + b1X1 + .. bnXn");
-       props2Help.put(LOGF0SOLUTION, "whether to use log(independent variable)");
-       props2Help.put(SOLUTIONSIZE, "size of the solution, number of dependend variables");
+	public String getName() {
+		return name;
+	}
 
-   }
-    
-  /**/
-  public boolean compute() throws Exception
-  {
-      FeatureFileReader featureFile = FeatureFileReader.getFeatureFileReader(getProp(FEATUREFILE));
-      UnitFileReader unitFile = new UnitFileReader(getProp(UNITFILE));
-      TimelineReader waveTimeline = null;
-      waveTimeline = new TimelineReader(getProp(WAVETIMELINE));
-      
-      PrintWriter toLeftFeaturesFile = new PrintWriter(new FileOutputStream(leftF0FeaturesFileName));
-      PrintWriter toMidFeaturesFile = new PrintWriter(new FileOutputStream(midF0FeaturesFileName));
-      PrintWriter toRightFeaturesFile = new PrintWriter(new FileOutputStream(rightF0FeaturesFileName));
-      
-      FeatureDefinition featureDefinition = featureFile.getFeatureDefinition();
-      
-      // Select the features that can be used for SFFS selection
-      String[] lingFactorsToSelect;
-      lingFactorsToSelect = selectLinguisticFactors(featureDefinition.getFeatureNames(), "Select linguistic factors for vowels:");
+	@Override
+	protected void initialiseComp() {
+		String rootDir = db.getProp(db.ROOTDIR);
+		String f0DirName = db.getProp(db.TEMPDIR);
+		this.f0Dir = new File(f0DirName);
+		if (!f0Dir.exists()) {
+			System.out.print("temp dir " + f0DirName + " does not exist; ");
+			if (!f0Dir.mkdir()) {
+				throw new Error("Could not create F0DIR");
+			}
+			System.out.print("Created successfully.\n");
+		}
+		this.leftF0FeaturesFileName = f0Dir + "f0.sop.left.feats";
+		this.midF0FeaturesFileName = f0Dir + "f0.sop.mid.feats";
+		this.rightF0FeaturesFileName = f0Dir + "f0.sop.right.feats";
+		this.interceptTerm = Boolean.valueOf(getProp(INTERCEPTTERM)).booleanValue();
+		this.logF0 = Boolean.valueOf(getProp(LOGF0SOLUTION)).booleanValue();
+		this.solutionSize = Integer.parseInt(getProp(SOLUTIONSIZE));
 
-      System.out.println("F0 CART trainer: exporting f0 features");      
-      byte isVowel = featureDefinition.getFeatureValueAsByte("ph_vc", "+");
-      int iVC = featureDefinition.getFeatureIndex("ph_vc");
-      int iSegsFromSylStart = featureDefinition.getFeatureIndex("segs_from_syl_start");
-      int iSegsFromSylEnd = featureDefinition.getFeatureIndex("segs_from_syl_end");
-      int iCVoiced = featureDefinition.getFeatureIndex("ph_cvox");
-      byte isCVoiced = featureDefinition.getFeatureValueAsByte("ph_cvox", "+");
+	}
 
-      int nSyllables = 0;
-      for (int i=0, len=unitFile.getNumberOfUnits(); i<len; i++) {
-          // We estimate that feature extraction takes 1/10 of the total time
-          // (that's probably wrong, but never mind)
-          percent = 10*i/len;
-          FeatureVector fv = featureFile.getFeatureVector(i);
-          if (fv.getByteFeature(iVC) == isVowel) {
-              // Found a vowel, i.e. found a syllable.
-              int mid = i;
-              FeatureVector fvMid = fv;
-              // Now find first/last voiced unit in the syllable:
-              int first = i;
-              for(int j=1, lookLeft = (int)fvMid.getByteFeature(iSegsFromSylStart); j<lookLeft; j++) {
-                  fv = featureFile.getFeatureVector(mid-j); // units are in sequential order
-                  // No need to check if there are any vowels to the left of this one,
-                  // because of the way we do the search in the top-level loop.
-                  if (fv.getByteFeature(iCVoiced) != isCVoiced) {
-                      break; // mid-j is not voiced
-                  }
-                  first = mid-j; // OK, the unit we are currently looking at is part of the voiced syllable section
-              }
-              int last = i;
-              for(int j=1, lookRight = (int)fvMid.getByteFeature(iSegsFromSylEnd); j<lookRight; j++) {
-                  fv = featureFile.getFeatureVector(mid+j); // units are in sequential order
+	public SortedMap<String, String> getDefaultProps(DatabaseLayout dbl) {
+		this.db = dbl;
+		if (props == null) {
+			props = new TreeMap<String, String>();
+			String filedir = db.getProp(db.FILEDIR);
+			String maryext = db.getProp(db.MARYEXT);
+			props.put(FEATUREDIR, db.getProp(db.ROOTDIR) + "phonefeatures" + System.getProperty("file.separator"));
+			props.put(LABELDIR, db.getProp(db.ROOTDIR) + "phonelab" + System.getProperty("file.separator"));
+			props.put(FEATUREFILE, filedir + "phoneFeatures" + maryext);
+			props.put(UNITFILE, filedir + "phoneUnits" + maryext);
+			props.put(WAVETIMELINE, db.getProp(db.FILEDIR) + "timeline_waveforms" + db.getProp(db.MARYEXT));
+			props.put(F0LeftSoPFILE, filedir + "f0.left.sop");
+			props.put(F0MidSoPFILE, filedir + "f0.mid.sop");
+			props.put(F0RightSoPFILE, filedir + "f0.right.sop");
+			props.put(INTERCEPTTERM, "true");
+			props.put(LOGF0SOLUTION, "false");
+			props.put(SOLUTIONSIZE, "5");
 
-                  if (fv.getByteFeature(iVC) != isVowel && fv.getByteFeature(iCVoiced) != isCVoiced) {
-                      break; // mid+j is not voiced
-                  }
-                  last = mid+j; // OK, the unit we are currently looking at is part of the voiced syllable section
-              }
-              // TODO: make this more robust, e.g. by fitting two straight lines to the data:
-              Datagram[] midDatagrams = waveTimeline.getDatagrams(unitFile.getUnit(mid), unitFile.getSampleRate());
-              Datagram[] leftDatagrams = waveTimeline.getDatagrams(unitFile.getUnit(first), unitFile.getSampleRate());
-              Datagram[] rightDatagrams = waveTimeline.getDatagrams(unitFile.getUnit(last), unitFile.getSampleRate());
-              if (midDatagrams != null && midDatagrams.length > 0
-                      && leftDatagrams != null && leftDatagrams.length > 0
-                      && rightDatagrams != null && rightDatagrams.length > 0) {
-                  double midF0 = waveTimeline.getSampleRate() / (float) midDatagrams[midDatagrams.length/2].getDuration();
-                  double leftF0 = waveTimeline.getSampleRate() / (float) leftDatagrams[0].getDuration();
-                  double rightF0 = waveTimeline.getSampleRate() / (float) rightDatagrams[rightDatagrams.length-1].getDuration();
-                  //System.out.format("Syllable at %d (length %d ): left = %.3f, mid = %.3f, right = %.3f\n", mid, (last-first+1), leftF0, midF0, rightF0);
+		}
+		return props;
+	}
 
-                  for(int j=0; j < lingFactorsToSelect.length; j++){
-                    byte feaVal = fvMid.getByteFeature(featureDefinition.getFeatureIndex(lingFactorsToSelect[j]));
-                    toLeftFeaturesFile.print(feaVal + " ");                    
-                    toMidFeaturesFile.print(feaVal + " ");                    
-                    toRightFeaturesFile.print(feaVal + " ");                    
-                  }
-                  // last column is the F0 value
-                  if(midF0 == Double.NEGATIVE_INFINITY || midF0 == Double.POSITIVE_INFINITY){
-                    //System.out.println("midDatagrams.length/2 = " + midDatagrams.length/2);
-                    //System.out.println("midDatagrams[midDatagrams.length/2].getDuration() = " + midDatagrams[midDatagrams.length/2].getDuration());
-                    System.out.format("Syllable at %d (length %d ): left = %.3f, mid = %.3f, right = %.3f ", mid, (last-first+1), leftF0, midF0, rightF0);                    
-                    midF0 = (leftF0 + rightF0) / 2.0;
-                    System.out.format("mindF0 is Nan --> changed to (leftF0 + rightF0) / 2.0 = %.3f\n", midF0);
-                  }
-     
-                  if(logF0){
-                    toLeftFeaturesFile.println(Math.log(leftF0));
-                    toMidFeaturesFile.println(Math.log(midF0));
-                    toRightFeaturesFile.println(Math.log(rightF0));
-                  } else {
-                    toLeftFeaturesFile.println(leftF0);
-                    toMidFeaturesFile.println(midF0);                    
-                    toRightFeaturesFile.println(rightF0);
-                  }
-                  nSyllables++;
-                  
-              }                 
-              // Skip the part we just covered:
-              i = last;               
-          }
-      }
-      toLeftFeaturesFile.close();
-      toMidFeaturesFile.close();
-      toRightFeaturesFile.close();
-      System.out.println("F0 features extracted for "+nSyllables+" syllables");
-      
-      
-      int cols, rows;
+	protected void setupHelp() {
+		props2Help = new TreeMap<String, String>();
+		props2Help.put(FEATUREDIR, "directory containing the phonefeatures");
+		props2Help.put(LABELDIR, "directory containing the phone label files");
+		props2Help.put(FEATUREFILE, "file containing all phone units and their target cost features");
+		props2Help.put(UNITFILE, "file containing all phone units");
+		props2Help.put(WAVETIMELINE, "file containing all waveforms or models that can genarate them");
+		props2Help.put(F0LeftSoPFILE, "file containing the f0 left SoP model. Will be created by this module");
+		props2Help.put(F0MidSoPFILE, "file containing the f0 mid SoP model. Will be created by this module");
+		props2Help.put(F0RightSoPFILE, "file containing the f0 right SoP model. Will be created by this module");
+		props2Help.put(INTERCEPTTERM, "whether to include interceptTerm (b0) on the solution equation : b0 + b1X1 + .. bnXn");
+		props2Help.put(LOGF0SOLUTION, "whether to use log(independent variable)");
+		props2Help.put(SOLUTIONSIZE, "size of the solution, number of dependend variables");
 
-      double percentToTrain = 0.9;
-           
-      
-      SFFS sffs = new SFFS(solutionSize, interceptTerm, logF0);
-      
-      
-      System.out.println("\n==================================\nProcessing Left F0:");
-      // the final regression will be saved in this file
-      PrintWriter toSopLeftFile = new PrintWriter(new FileOutputStream(getProp(F0LeftSoPFILE)));      
-      // Save first the features definition on the output file
-      featureDefinition.writeTo(toSopLeftFile, false);
-      toSopLeftFile.println();      
-      SoP sopLeft = new SoP(featureDefinition);
-      sffs.trainModel(lingFactorsToSelect, leftF0FeaturesFileName, nSyllables, percentToTrain, sopLeft);
-      toSopLeftFile.println("f0.left");
-      sopLeft.saveSelectedFeatures(toSopLeftFile);
-      toSopLeftFile.close();
-      
-      
-      System.out.println("\n==================================\nProcessing Mid F0:");
-      // the final regression will be saved in this file
-      PrintWriter toSopMidFile = new PrintWriter(new FileOutputStream(getProp(F0MidSoPFILE)));      
-      // Save first the features definition on the output file
-      featureDefinition.writeTo(toSopMidFile, false);
-      toSopMidFile.println();            
-      SoP sopMid = new SoP(featureDefinition);
-      sffs.trainModel(lingFactorsToSelect, midF0FeaturesFileName, nSyllables, percentToTrain, sopMid);
-      toSopMidFile.println("f0.mid");
-      sopMid.saveSelectedFeatures(toSopMidFile);
-      toSopMidFile.close();
-      
-      
-      System.out.println("\n==================================\nProcessing Right F0:");
-      //the final regression will be saved in this file
-      PrintWriter toSopRightFile = new PrintWriter(new FileOutputStream(getProp(F0RightSoPFILE)));      
-      // Save first the features definition on the output file
-      featureDefinition.writeTo(toSopRightFile, false);
-      toSopRightFile.println();
-      SoP sopRight = new SoP(featureDefinition);
-      sffs.trainModel(lingFactorsToSelect, rightF0FeaturesFileName, nSyllables, percentToTrain, sopRight);
-      toSopRightFile.println("f0.right");
-      sopRight.saveSelectedFeatures(toSopRightFile);
-      toSopRightFile.close();
-      
-      
-      percent = 100;
+	}
 
-      return true;
-  }
-  
+	/**/
+	public boolean compute() throws Exception {
+		FeatureFileReader featureFile = FeatureFileReader.getFeatureFileReader(getProp(FEATUREFILE));
+		UnitFileReader unitFile = new UnitFileReader(getProp(UNITFILE));
+		TimelineReader waveTimeline = null;
+		waveTimeline = new TimelineReader(getProp(WAVETIMELINE));
 
-  
-  public String[] selectLinguisticFactors(String featureNames, String label) throws IOException
-  {
-      String[] lingFactors=null;
-      String features = checkFeatureList(featureNames);
- 
-      final JFrame frame = new JFrame(label);
-      GridBagLayout gridBagLayout = new GridBagLayout();
-      GridBagConstraints gridC = new GridBagConstraints();
-      frame.getContentPane().setLayout( gridBagLayout );
-      
-      final JEditorPane editPane = new JEditorPane();
-      editPane.setPreferredSize(new Dimension(500, 500));
-      editPane.setText(features);        
-      
-      JButton saveButton = new JButton("Save");
-      saveButton.addActionListener(new ActionListener() {
-         public void actionPerformed(ActionEvent e) {                
-             setSuccess(true);
-             frame.setVisible(false);
-          }
-      });
-      JButton cancelButton = new JButton("Cancel");
-      cancelButton.addActionListener(new ActionListener() {
-          public void actionPerformed(ActionEvent e) {
-              setSuccess(false);
-              frame.setVisible(false);
-          }
-      });
-      
-      gridC.gridx = 0;
-      gridC.gridy = 0;
-      // resize scroll pane:
-      gridC.weightx = 1;
-      gridC.weighty = 1;
-      gridC.fill = GridBagConstraints.HORIZONTAL;
-      JScrollPane scrollPane = new JScrollPane(editPane);
-      scrollPane.setPreferredSize(editPane.getPreferredSize());
-      gridBagLayout.setConstraints( scrollPane, gridC );
-      frame.getContentPane().add(scrollPane);
-      gridC.gridy = 1;
-      // do not resize buttons:
-      gridC.weightx = 0;
-      gridC.weighty = 0;
-      JPanel buttonPanel = new JPanel();
-      buttonPanel.setLayout(new FlowLayout());
-      buttonPanel.add(saveButton);
-      buttonPanel.add(cancelButton);
-      gridBagLayout.setConstraints( buttonPanel, gridC );
-      frame.getContentPane().add(buttonPanel);
-      frame.pack();
-      frame.setVisible(true);
-      
-      do {
-          try {
-              Thread.sleep(10); 
-          } catch (InterruptedException e) {}
-      } while (frame.isVisible());
-      frame.dispose();
+		PrintWriter toLeftFeaturesFile = new PrintWriter(new FileOutputStream(leftF0FeaturesFileName));
+		PrintWriter toMidFeaturesFile = new PrintWriter(new FileOutputStream(midF0FeaturesFileName));
+		PrintWriter toRightFeaturesFile = new PrintWriter(new FileOutputStream(rightF0FeaturesFileName));
 
-      if (success) {
-          try{
-              lingFactors = saveFeatures(editPane.getText());                
-          } catch (Exception ex){
-              ex.printStackTrace();
-              throw new Error("Error defining replacements");
-          }
-      }
-     //return true;
-      return lingFactors;
-  }
+		FeatureDefinition featureDefinition = featureFile.getFeatureDefinition();
 
-  private String checkFeatureList(String featureNames) throws IOException
-  {
-      String featureList = "";
-      String recommendedFeatureList = "";
-      String feaList[] = featureNames.split(" ");
-      String line;
-     
-      for(int i=0; i<feaList.length; i++){
-          line = feaList[i];
-             
-          
-            // Exclude phone and phonological, those are by default used in makeLabes and makeQuestions
-            // Also exclude the halfphone features not used in HMM voices
-            // use this when finding the features that better predict f0, without including phonological features
-            if(! (line.contains("_vc") ||
-                  line.contains("_vlng") ||
-                  line.contains("_vheight") ||
-                  line.contains("_vfront") ||
-                  line.contains("_vrnd") ||
-                  line.contains("_ctype") ||
-                  line.contains("_cplace") ||
-                  line.contains("_cvox") ||
-                  line.contains("_phone") || 
-                  line.contains("ph_") ||
-                  line.contains("halfphone_") ||
-                  line.contentEquals("phone") )  ) {
-                featureList += line + "\n";
-            }
-        }
-      //return recommendedFeatureList + "\n" + featureList;
-      return featureList;
-      //return "";
+		// Select the features that can be used for SFFS selection
+		String[] lingFactorsToSelect;
+		lingFactorsToSelect = selectLinguisticFactors(featureDefinition.getFeatureNames(),
+				"Select linguistic factors for vowels:");
 
-  }
- 
-  protected void setSuccess(boolean val)
-  {
-      success = val;
-  }  
-  
-  private String[] saveFeatures(String newFeatures)
-  {     
-     String fea[] = newFeatures.split("\n");
-     String[] lingFactors = new String[fea.length];
-     System.out.print("Selected linguistic factors (" + fea.length + "):");
-     for(int i=0; i<fea.length; i++){
-       System.out.print(fea[i] + " ");
-       lingFactors[i] = fea[i];
-     }     
-     System.out.println();
-     return lingFactors;
-  }  
-   
-  /**
-   * Provide the progress of computation, in percent, or -1 if
-   * that feature is not implemented.
-   * @return -1 if not implemented, or an integer between 0 and 100.
-   */
-  public int getProgress()
-  {
-      return percent;
-  }
+		System.out.println("F0 CART trainer: exporting f0 features");
+		byte isVowel = featureDefinition.getFeatureValueAsByte("ph_vc", "+");
+		int iVC = featureDefinition.getFeatureIndex("ph_vc");
+		int iSegsFromSylStart = featureDefinition.getFeatureIndex("segs_from_syl_start");
+		int iSegsFromSylEnd = featureDefinition.getFeatureIndex("segs_from_syl_end");
+		int iCVoiced = featureDefinition.getFeatureIndex("ph_cvox");
+		byte isCVoiced = featureDefinition.getFeatureValueAsByte("ph_cvox", "+");
 
-  public static void main(String[] args) throws Exception
-  {
-      F0CARTTrainer f0ct = new F0CARTTrainer();
-      DatabaseLayout db = new DatabaseLayout(f0ct);
-      f0ct.compute();
-  }
+		int nSyllables = 0;
+		for (int i = 0, len = unitFile.getNumberOfUnits(); i < len; i++) {
+			// We estimate that feature extraction takes 1/10 of the total time
+			// (that's probably wrong, but never mind)
+			percent = 10 * i / len;
+			FeatureVector fv = featureFile.getFeatureVector(i);
+			if (fv.getByteFeature(iVC) == isVowel) {
+				// Found a vowel, i.e. found a syllable.
+				int mid = i;
+				FeatureVector fvMid = fv;
+				// Now find first/last voiced unit in the syllable:
+				int first = i;
+				for (int j = 1, lookLeft = (int) fvMid.getByteFeature(iSegsFromSylStart); j < lookLeft; j++) {
+					fv = featureFile.getFeatureVector(mid - j); // units are in sequential order
+					// No need to check if there are any vowels to the left of this one,
+					// because of the way we do the search in the top-level loop.
+					if (fv.getByteFeature(iCVoiced) != isCVoiced) {
+						break; // mid-j is not voiced
+					}
+					first = mid - j; // OK, the unit we are currently looking at is part of the voiced syllable section
+				}
+				int last = i;
+				for (int j = 1, lookRight = (int) fvMid.getByteFeature(iSegsFromSylEnd); j < lookRight; j++) {
+					fv = featureFile.getFeatureVector(mid + j); // units are in sequential order
+
+					if (fv.getByteFeature(iVC) != isVowel && fv.getByteFeature(iCVoiced) != isCVoiced) {
+						break; // mid+j is not voiced
+					}
+					last = mid + j; // OK, the unit we are currently looking at is part of the voiced syllable section
+				}
+				// TODO: make this more robust, e.g. by fitting two straight lines to the data:
+				Datagram[] midDatagrams = waveTimeline.getDatagrams(unitFile.getUnit(mid), unitFile.getSampleRate());
+				Datagram[] leftDatagrams = waveTimeline.getDatagrams(unitFile.getUnit(first), unitFile.getSampleRate());
+				Datagram[] rightDatagrams = waveTimeline.getDatagrams(unitFile.getUnit(last), unitFile.getSampleRate());
+				if (midDatagrams != null && midDatagrams.length > 0 && leftDatagrams != null && leftDatagrams.length > 0
+						&& rightDatagrams != null && rightDatagrams.length > 0) {
+					double midF0 = waveTimeline.getSampleRate() / (float) midDatagrams[midDatagrams.length / 2].getDuration();
+					double leftF0 = waveTimeline.getSampleRate() / (float) leftDatagrams[0].getDuration();
+					double rightF0 = waveTimeline.getSampleRate()
+							/ (float) rightDatagrams[rightDatagrams.length - 1].getDuration();
+					// System.out.format("Syllable at %d (length %d ): left = %.3f, mid = %.3f, right = %.3f\n", mid,
+					// (last-first+1), leftF0, midF0, rightF0);
+
+					for (int j = 0; j < lingFactorsToSelect.length; j++) {
+						byte feaVal = fvMid.getByteFeature(featureDefinition.getFeatureIndex(lingFactorsToSelect[j]));
+						toLeftFeaturesFile.print(feaVal + " ");
+						toMidFeaturesFile.print(feaVal + " ");
+						toRightFeaturesFile.print(feaVal + " ");
+					}
+					// last column is the F0 value
+					if (midF0 == Double.NEGATIVE_INFINITY || midF0 == Double.POSITIVE_INFINITY) {
+						// System.out.println("midDatagrams.length/2 = " + midDatagrams.length/2);
+						// System.out.println("midDatagrams[midDatagrams.length/2].getDuration() = " +
+						// midDatagrams[midDatagrams.length/2].getDuration());
+						System.out.format("Syllable at %d (length %d ): left = %.3f, mid = %.3f, right = %.3f ", mid, (last
+								- first + 1), leftF0, midF0, rightF0);
+						midF0 = (leftF0 + rightF0) / 2.0;
+						System.out.format("mindF0 is Nan --> changed to (leftF0 + rightF0) / 2.0 = %.3f\n", midF0);
+					}
+
+					if (logF0) {
+						toLeftFeaturesFile.println(Math.log(leftF0));
+						toMidFeaturesFile.println(Math.log(midF0));
+						toRightFeaturesFile.println(Math.log(rightF0));
+					} else {
+						toLeftFeaturesFile.println(leftF0);
+						toMidFeaturesFile.println(midF0);
+						toRightFeaturesFile.println(rightF0);
+					}
+					nSyllables++;
+
+				}
+				// Skip the part we just covered:
+				i = last;
+			}
+		}
+		toLeftFeaturesFile.close();
+		toMidFeaturesFile.close();
+		toRightFeaturesFile.close();
+		System.out.println("F0 features extracted for " + nSyllables + " syllables");
+
+		int cols, rows;
+
+		double percentToTrain = 0.9;
+
+		SFFS sffs = new SFFS(solutionSize, interceptTerm, logF0);
+
+		System.out.println("\n==================================\nProcessing Left F0:");
+		// the final regression will be saved in this file
+		PrintWriter toSopLeftFile = new PrintWriter(new FileOutputStream(getProp(F0LeftSoPFILE)));
+		// Save first the features definition on the output file
+		featureDefinition.writeTo(toSopLeftFile, false);
+		toSopLeftFile.println();
+		SoP sopLeft = new SoP(featureDefinition);
+		sffs.trainModel(lingFactorsToSelect, leftF0FeaturesFileName, nSyllables, percentToTrain, sopLeft);
+		toSopLeftFile.println("f0.left");
+		sopLeft.saveSelectedFeatures(toSopLeftFile);
+		toSopLeftFile.close();
+
+		System.out.println("\n==================================\nProcessing Mid F0:");
+		// the final regression will be saved in this file
+		PrintWriter toSopMidFile = new PrintWriter(new FileOutputStream(getProp(F0MidSoPFILE)));
+		// Save first the features definition on the output file
+		featureDefinition.writeTo(toSopMidFile, false);
+		toSopMidFile.println();
+		SoP sopMid = new SoP(featureDefinition);
+		sffs.trainModel(lingFactorsToSelect, midF0FeaturesFileName, nSyllables, percentToTrain, sopMid);
+		toSopMidFile.println("f0.mid");
+		sopMid.saveSelectedFeatures(toSopMidFile);
+		toSopMidFile.close();
+
+		System.out.println("\n==================================\nProcessing Right F0:");
+		// the final regression will be saved in this file
+		PrintWriter toSopRightFile = new PrintWriter(new FileOutputStream(getProp(F0RightSoPFILE)));
+		// Save first the features definition on the output file
+		featureDefinition.writeTo(toSopRightFile, false);
+		toSopRightFile.println();
+		SoP sopRight = new SoP(featureDefinition);
+		sffs.trainModel(lingFactorsToSelect, rightF0FeaturesFileName, nSyllables, percentToTrain, sopRight);
+		toSopRightFile.println("f0.right");
+		sopRight.saveSelectedFeatures(toSopRightFile);
+		toSopRightFile.close();
+
+		percent = 100;
+
+		return true;
+	}
+
+	public String[] selectLinguisticFactors(String featureNames, String label) throws IOException {
+		String[] lingFactors = null;
+		String features = checkFeatureList(featureNames);
+
+		final JFrame frame = new JFrame(label);
+		GridBagLayout gridBagLayout = new GridBagLayout();
+		GridBagConstraints gridC = new GridBagConstraints();
+		frame.getContentPane().setLayout(gridBagLayout);
+
+		final JEditorPane editPane = new JEditorPane();
+		editPane.setPreferredSize(new Dimension(500, 500));
+		editPane.setText(features);
+
+		JButton saveButton = new JButton("Save");
+		saveButton.addActionListener(new ActionListener() {
+			public void actionPerformed(ActionEvent e) {
+				setSuccess(true);
+				frame.setVisible(false);
+			}
+		});
+		JButton cancelButton = new JButton("Cancel");
+		cancelButton.addActionListener(new ActionListener() {
+			public void actionPerformed(ActionEvent e) {
+				setSuccess(false);
+				frame.setVisible(false);
+			}
+		});
+
+		gridC.gridx = 0;
+		gridC.gridy = 0;
+		// resize scroll pane:
+		gridC.weightx = 1;
+		gridC.weighty = 1;
+		gridC.fill = GridBagConstraints.HORIZONTAL;
+		JScrollPane scrollPane = new JScrollPane(editPane);
+		scrollPane.setPreferredSize(editPane.getPreferredSize());
+		gridBagLayout.setConstraints(scrollPane, gridC);
+		frame.getContentPane().add(scrollPane);
+		gridC.gridy = 1;
+		// do not resize buttons:
+		gridC.weightx = 0;
+		gridC.weighty = 0;
+		JPanel buttonPanel = new JPanel();
+		buttonPanel.setLayout(new FlowLayout());
+		buttonPanel.add(saveButton);
+		buttonPanel.add(cancelButton);
+		gridBagLayout.setConstraints(buttonPanel, gridC);
+		frame.getContentPane().add(buttonPanel);
+		frame.pack();
+		frame.setVisible(true);
+
+		do {
+			try {
+				Thread.sleep(10);
+			} catch (InterruptedException e) {
+			}
+		} while (frame.isVisible());
+		frame.dispose();
+
+		if (success) {
+			try {
+				lingFactors = saveFeatures(editPane.getText());
+			} catch (Exception ex) {
+				ex.printStackTrace();
+				throw new Error("Error defining replacements");
+			}
+		}
+		// return true;
+		return lingFactors;
+	}
+
+	private String checkFeatureList(String featureNames) throws IOException {
+		String featureList = "";
+		String recommendedFeatureList = "";
+		String feaList[] = featureNames.split(" ");
+		String line;
+
+		for (int i = 0; i < feaList.length; i++) {
+			line = feaList[i];
+
+			// Exclude phone and phonological, those are by default used in makeLabes and makeQuestions
+			// Also exclude the halfphone features not used in HMM voices
+			// use this when finding the features that better predict f0, without including phonological features
+			if (!(line.contains("_vc") || line.contains("_vlng") || line.contains("_vheight") || line.contains("_vfront")
+					|| line.contains("_vrnd") || line.contains("_ctype") || line.contains("_cplace") || line.contains("_cvox")
+					|| line.contains("_phone") || line.contains("ph_") || line.contains("halfphone_") || line
+						.contentEquals("phone"))) {
+				featureList += line + "\n";
+			}
+		}
+		// return recommendedFeatureList + "\n" + featureList;
+		return featureList;
+		// return "";
+
+	}
+
+	protected void setSuccess(boolean val) {
+		success = val;
+	}
+
+	private String[] saveFeatures(String newFeatures) {
+		String fea[] = newFeatures.split("\n");
+		String[] lingFactors = new String[fea.length];
+		System.out.print("Selected linguistic factors (" + fea.length + "):");
+		for (int i = 0; i < fea.length; i++) {
+			System.out.print(fea[i] + " ");
+			lingFactors[i] = fea[i];
+		}
+		System.out.println();
+		return lingFactors;
+	}
+
+	/**
+	 * Provide the progress of computation, in percent, or -1 if that feature is not implemented.
+	 * 
+	 * @return -1 if not implemented, or an integer between 0 and 100.
+	 */
+	public int getProgress() {
+		return percent;
+	}
+
+	public static void main(String[] args) throws Exception {
+		F0CARTTrainer f0ct = new F0CARTTrainer();
+		DatabaseLayout db = new DatabaseLayout(f0ct);
+		f0ct.compute();
+	}
 
 }
-

marytts-builder/src/main/java/marytts/tools/voiceimport/FeatureSelection.java

@@ -42,194 +42,180 @@ import marytts.client.http.MaryHttpClient;
 import marytts.util.http.Address;
 import marytts.util.io.FileUtils;
 
-
-
 /**
- * For the given texts, compute unit features and align them
- * with the given unit labels.
+ * For the given texts, compute unit features and align them with the given unit labels.
+ * 
  * @author schroed
  *
  */
-public class FeatureSelection extends VoiceImportComponent
-{
-    protected File featureFile;
-    protected String features;
-    protected String locale;
-    protected MaryHttpClient mary;
-    protected boolean success = true;
-    
-    protected DatabaseLayout db = null;
-    
-    public String FEATUREFILE = "FeatureSelection.featureFile";
-    public String MARYSERVERHOST = "FeatureSelection.maryServerHost";
-    public String MARYSERVERPORT = "FeatureSelection.maryServerPort";
-       
-   
-    
-    public String getName(){
-        return "FeatureSelection";
-    }
-    
-    
-    @Override
-    protected void initialiseComp()
-    {      
-        locale = db.getProp(db.LOCALE);   
-        
-        mary = null; // initialised only if needed   
-        featureFile = new File(getProp(FEATUREFILE));
-        if (featureFile.exists()) {
-            System.out.println("Loading features from file "+getProp(FEATUREFILE));
-            try {
-                features = FileUtils.getFileAsString(featureFile, "UTF-8");
-            } catch (IOException e) {
-                e.printStackTrace();
-            }
-        }
-    }
-     
-     public SortedMap<String,String> getDefaultProps(DatabaseLayout theDb){
-         this.db = theDb;
-         if (props == null){
-             props = new TreeMap<String, String>();
-             props.put(FEATUREFILE,
-                     db.getProp(db.CONFIGDIR) + "features.txt");
-             props.put(MARYSERVERHOST,"localhost");
-             props.put(MARYSERVERPORT,"59125");
-         } 
-         return props;
-     }
-     
-     protected void setupHelp() {
-         props2Help = new TreeMap<String, String>();
-         props2Help.put(FEATUREFILE, "file containing the list of features to use." 
-                 +"Will be created by querying MARY server if it does not exist");
-         props2Help.put(MARYSERVERHOST,"the host were the Mary server is running, default: \"localhost\"");
-         props2Help.put(MARYSERVERPORT,"the port were the Mary server is listening, default: \"59125\"");
-     }
-     
-     public MaryHttpClient getMaryClient() throws IOException
-     {
-        if (mary == null) {
-            try{
-                mary = new MaryHttpClient(new Address(getProp(MARYSERVERHOST), Integer.parseInt(getProp(MARYSERVERPORT))));        
-            } catch (IOException e){
-                throw new IOException("Could not connect to Maryserver at "
-                        +getProp(MARYSERVERHOST)+" "+getProp(MARYSERVERPORT));
-            }
-        }
-        return mary;
-    }
-     
-     protected void saveFeatures(String newFeatures)
-     {
-         System.out.println("Saving features to "+featureFile.getAbsolutePath());
-         features = newFeatures;
-         if (!features.contains(PhoneUnitFeatureComputer.PHONEFEATURE)) {
-             JOptionPane.showMessageDialog(null, "The features '"+PhoneUnitFeatureComputer.PHONEFEATURE
-                     +"' is not present.\nThis will lead to problems in the further processing.",
-                     "Important feature missing", JOptionPane.WARNING_MESSAGE);
-         }
-         try {
-             PrintWriter pw = new PrintWriter(featureFile, "UTF-8");
-             pw.println(features);
-             pw.close();
-         } catch (IOException e) {
-             System.err.println("Cannot save features:");
-             e.printStackTrace();
-             success = false;
-         }
-     }
-     
-     protected void setSuccess(boolean val)
-     {
-         success = val;
-     }
+public class FeatureSelection extends VoiceImportComponent {
+	protected File featureFile;
+	protected String features;
+	protected String locale;
+	protected MaryHttpClient mary;
+	protected boolean success = true;
 
-    public boolean compute() throws IOException
-    {
-        if (features == null) {
-            mary = getMaryClient();
-            features = mary.getDiscreteFeatures(locale);
-            features = features.replaceAll(" ", "\n");
-        }
+	protected DatabaseLayout db = null;
 
-        final JFrame frame = new JFrame("Features to use for building voice");
-        GridBagLayout gridBagLayout = new GridBagLayout();
-        GridBagConstraints gridC = new GridBagConstraints();
-        frame.getContentPane().setLayout( gridBagLayout );
-        
-        final JEditorPane editPane = new JEditorPane();
-        editPane.setPreferredSize(new Dimension(500, 500));
-        editPane.setText(features);        
-        
-        JButton saveButton = new JButton("Save");
-        saveButton.addActionListener(new ActionListener() {
-           public void actionPerformed(ActionEvent e) {                
-               setSuccess(true);
-               frame.setVisible(false);
-            }
-        });
-        JButton cancelButton = new JButton("Cancel");
-        cancelButton.addActionListener(new ActionListener() {
-            public void actionPerformed(ActionEvent e) {
-                setSuccess(false);
-                frame.setVisible(false);
-            }
-        });
-        
-        gridC.gridx = 0;
-        gridC.gridy = 0;
-        // resize scroll pane:
-        gridC.weightx = 1;
-        gridC.weighty = 1;
-        gridC.fill = GridBagConstraints.HORIZONTAL;
-        JScrollPane scrollPane = new JScrollPane(editPane);
-        scrollPane.setPreferredSize(editPane.getPreferredSize());
-        gridBagLayout.setConstraints( scrollPane, gridC );
-        frame.getContentPane().add(scrollPane);
-        gridC.gridy = 1;
-        // do not resize buttons:
-        gridC.weightx = 0;
-        gridC.weighty = 0;
-        JPanel buttonPanel = new JPanel();
-        buttonPanel.setLayout(new FlowLayout());
-        buttonPanel.add(saveButton);
-        buttonPanel.add(cancelButton);
-        gridBagLayout.setConstraints( buttonPanel, gridC );
-        frame.getContentPane().add(buttonPanel);
-        frame.pack();
-        frame.setVisible(true);
-        
-        do {
-            try {
-                Thread.sleep(10); 
-            } catch (InterruptedException e) {}
-        } while (frame.isVisible());
-        frame.dispose();
+	public String FEATUREFILE = "FeatureSelection.featureFile";
+	public String MARYSERVERHOST = "FeatureSelection.maryServerHost";
+	public String MARYSERVERPORT = "FeatureSelection.maryServerPort";
 
-        if (success) {
-            try{
-                saveFeatures(editPane.getText());                
-            } catch (Exception ex){
-                ex.printStackTrace();
-                throw new Error("Error defining replacements");
-            }
-        }
+	public String getName() {
+		return "FeatureSelection";
+	}
 
-        return success;
-    }
+	@Override
+	protected void initialiseComp() {
+		locale = db.getProp(db.LOCALE);
 
+		mary = null; // initialised only if needed
+		featureFile = new File(getProp(FEATUREFILE));
+		if (featureFile.exists()) {
+			System.out.println("Loading features from file " + getProp(FEATUREFILE));
+			try {
+				features = FileUtils.getFileAsString(featureFile, "UTF-8");
+			} catch (IOException e) {
+				e.printStackTrace();
+			}
+		}
+	}
 
-    /**
-     * Provide the progress of computation, in percent, or -1 if
-     * that feature is not implemented.
-     * @return -1 if not implemented, or an integer between 0 and 100.
-     */
-    public int getProgress()
-    {
-        return -1;
-    }
+	public SortedMap<String, String> getDefaultProps(DatabaseLayout theDb) {
+		this.db = theDb;
+		if (props == null) {
+			props = new TreeMap<String, String>();
+			props.put(FEATUREFILE, db.getProp(db.CONFIGDIR) + "features.txt");
+			props.put(MARYSERVERHOST, "localhost");
+			props.put(MARYSERVERPORT, "59125");
+		}
+		return props;
+	}
+
+	protected void setupHelp() {
+		props2Help = new TreeMap<String, String>();
+		props2Help.put(FEATUREFILE, "file containing the list of features to use."
+				+ "Will be created by querying MARY server if it does not exist");
+		props2Help.put(MARYSERVERHOST, "the host were the Mary server is running, default: \"localhost\"");
+		props2Help.put(MARYSERVERPORT, "the port were the Mary server is listening, default: \"59125\"");
+	}
+
+	public MaryHttpClient getMaryClient() throws IOException {
+		if (mary == null) {
+			try {
+				mary = new MaryHttpClient(new Address(getProp(MARYSERVERHOST), Integer.parseInt(getProp(MARYSERVERPORT))));
+			} catch (IOException e) {
+				throw new IOException("Could not connect to Maryserver at " + getProp(MARYSERVERHOST) + " "
+						+ getProp(MARYSERVERPORT));
+			}
+		}
+		return mary;
+	}
+
+	protected void saveFeatures(String newFeatures) {
+		System.out.println("Saving features to " + featureFile.getAbsolutePath());
+		features = newFeatures;
+		if (!features.contains(PhoneUnitFeatureComputer.PHONEFEATURE)) {
+			JOptionPane.showMessageDialog(null, "The features '" + PhoneUnitFeatureComputer.PHONEFEATURE
+					+ "' is not present.\nThis will lead to problems in the further processing.", "Important feature missing",
+					JOptionPane.WARNING_MESSAGE);
+		}
+		try {
+			PrintWriter pw = new PrintWriter(featureFile, "UTF-8");
+			pw.println(features);
+			pw.close();
+		} catch (IOException e) {
+			System.err.println("Cannot save features:");
+			e.printStackTrace();
+			success = false;
+		}
+	}
+
+	protected void setSuccess(boolean val) {
+		success = val;
+	}
+
+	public boolean compute() throws IOException {
+		if (features == null) {
+			mary = getMaryClient();
+			features = mary.getDiscreteFeatures(locale);
+			features = features.replaceAll(" ", "\n");
+		}
+
+		final JFrame frame = new JFrame("Features to use for building voice");
+		GridBagLayout gridBagLayout = new GridBagLayout();
+		GridBagConstraints gridC = new GridBagConstraints();
+		frame.getContentPane().setLayout(gridBagLayout);
+
+		final JEditorPane editPane = new JEditorPane();
+		editPane.setPreferredSize(new Dimension(500, 500));
+		editPane.setText(features);
+
+		JButton saveButton = new JButton("Save");
+		saveButton.addActionListener(new ActionListener() {
+			public void actionPerformed(ActionEvent e) {
+				setSuccess(true);
+				frame.setVisible(false);
+			}
+		});
+		JButton cancelButton = new JButton("Cancel");
+		cancelButton.addActionListener(new ActionListener() {
+			public void actionPerformed(ActionEvent e) {
+				setSuccess(false);
+				frame.setVisible(false);
+			}
+		});
+
+		gridC.gridx = 0;
+		gridC.gridy = 0;
+		// resize scroll pane:
+		gridC.weightx = 1;
+		gridC.weighty = 1;
+		gridC.fill = GridBagConstraints.HORIZONTAL;
+		JScrollPane scrollPane = new JScrollPane(editPane);
+		scrollPane.setPreferredSize(editPane.getPreferredSize());
+		gridBagLayout.setConstraints(scrollPane, gridC);
+		frame.getContentPane().add(scrollPane);
+		gridC.gridy = 1;
+		// do not resize buttons:
+		gridC.weightx = 0;
+		gridC.weighty = 0;
+		JPanel buttonPanel = new JPanel();
+		buttonPanel.setLayout(new FlowLayout());
+		buttonPanel.add(saveButton);
+		buttonPanel.add(cancelButton);
+		gridBagLayout.setConstraints(buttonPanel, gridC);
+		frame.getContentPane().add(buttonPanel);
+		frame.pack();
+		frame.setVisible(true);
+
+		do {
+			try {
+				Thread.sleep(10);
+			} catch (InterruptedException e) {
+			}
+		} while (frame.isVisible());
+		frame.dispose();
+
+		if (success) {
+			try {
+				saveFeatures(editPane.getText());
+			} catch (Exception ex) {
+				ex.printStackTrace();
+				throw new Error("Error defining replacements");
+			}
+		}
+
+		return success;
+	}
+
+	/**
+	 * Provide the progress of computation, in percent, or -1 if that feature is not implemented.
+	 * 
+	 * @return -1 if not implemented, or an integer between 0 and 100.
+	 */
+	public int getProgress() {
+		return -1;
+	}
 
 }
-

marytts-builder/src/main/java/marytts/tools/voiceimport/HMMParameterExtractor.java

@@ -36,249 +36,252 @@ import marytts.htsengine.HTSUttModel;
 import marytts.htsengine.HTSVocoder;
 import marytts.modules.HTSEngine;
 
-
 /**
  * A component for extracting mfccs, lab, state labels, and hmm generated waves (options given to choose)
+ * 
  * @author Marcela, Sathish Pammi
  *
  */
 public class HMMParameterExtractor extends VoiceImportComponent {
 
-    public final String MARYBASE = "HMMParameterExtractor.maryBase";
-    public final String VOICECONFIG = "HMMParameterExtractor.voiceConfigFile";
-    public final String VOICENAME = "HMMParameterExtractor.voiceName";
-    public final String PHONEFEATS = "HMMParameterExtractor.phonefeaturesDir";
-    public final String OUTHMMDIR = "HMMParameterExtractor.outputDir";
-    public final String PPARAMETERS = "HMMParameterExtractor.printParameters";
-    public final String PLAB = "HMMParameterExtractor.printLab";
-    public final String PWAVE = "HMMParameterExtractor.printWave";
-    public final String PSLAB = "HMMParameterExtractor.PrintStateLab";
-    public final String USEGV   = "HMMParameterExtractor.useGV";
-    
-    private DatabaseLayout db;    
-    private String sCostDirectory;
-    private String mfccExt = ".mfcc";
-    private String pfeatExt = ".pfeats";
-    private String slabExt = ".slab";
-    private String labExt = ".lab";
-    private String wavExt = ".wav";
-    private String hplabExt = ".hplab";
-    protected int percent = 0;
-    private HTSEngine hmm_tts;
-    private HMMData htsData;
-    
-    protected void setupHelp() {
-        props2Help = new TreeMap();
-        props2Help.put(MARYBASE,"Mary Base.  ex: /home/user/MARY400/ ");
-        props2Help.put(VOICECONFIG,"locale ex: english"); 
-        props2Help.put(VOICENAME, "Voice name ex: slt-arctic");
-        props2Help.put(PHONEFEATS, "Phonefeatures directory");
-        props2Help.put(OUTHMMDIR, "Output directory to store hmm generated parameters");
-        props2Help.put(PPARAMETERS, "Generate parameters like MFCC, PITCH files?");
-        props2Help.put(PLAB, "Generate HMM Label files?");
-        props2Help.put(PWAVE, "Generate HMM WAVE files?");
-        props2Help.put(PSLAB, "Generate HMM State Label files?");
-        props2Help.put(USEGV, "Use Global variance(GV) in parameter generation?");
-    }
-    
-    public SortedMap<String, String> getDefaultProps(DatabaseLayout db) {
-        this.db = db;
-        if (props == null){
-            props = new TreeMap();
-            props.put(MARYBASE,db.getProp(db.MARYBASE));
-            props.put(VOICECONFIG,"english-hsmm-slt.config"); 
-            props.put(VOICENAME, "hsmm-slt");
-            props.put(PHONEFEATS,db.getProp(db.ROOTDIR)+File.separator+"phonefeatures");
-            props.put(OUTHMMDIR, db.getProp(db.ROOTDIR)+File.separator+"sCost"+File.separator+"hmmparams"); 
-            props.put(PPARAMETERS,"true");
-            props.put(PLAB,"true");
-            props.put(PWAVE,"false");
-            props.put(PSLAB,"false");
-            props.put(USEGV,"false");
-        }
-        return props;
-    }
+	public final String MARYBASE = "HMMParameterExtractor.maryBase";
+	public final String VOICECONFIG = "HMMParameterExtractor.voiceConfigFile";
+	public final String VOICENAME = "HMMParameterExtractor.voiceName";
+	public final String PHONEFEATS = "HMMParameterExtractor.phonefeaturesDir";
+	public final String OUTHMMDIR = "HMMParameterExtractor.outputDir";
+	public final String PPARAMETERS = "HMMParameterExtractor.printParameters";
+	public final String PLAB = "HMMParameterExtractor.printLab";
+	public final String PWAVE = "HMMParameterExtractor.printWave";
+	public final String PSLAB = "HMMParameterExtractor.PrintStateLab";
+	public final String USEGV = "HMMParameterExtractor.useGV";
 
-    public final String getName() {
-        return "HMMParameterExtractor";
-    }
-    
-    @Override
-    protected void initialiseComp() {
-        // sCost dir creation, if doesn't exists
-        sCostDirectory = db.getProp(db.ROOTDIR)+File.separator+"sCost";
-        File sCostDir =  new File(sCostDirectory);
-        if(!sCostDir.exists()){
-            System.out.print(sCostDir.getAbsolutePath()+" does not exist; ");
-            if (!sCostDir.mkdir()){
-                throw new Error("Could not create "+sCostDir.getAbsolutePath());
-            }
-            System.out.print("Created successfully.\n");
-        }
-    }
+	private DatabaseLayout db;
+	private String sCostDirectory;
+	private String mfccExt = ".mfcc";
+	private String pfeatExt = ".pfeats";
+	private String slabExt = ".slab";
+	private String labExt = ".lab";
+	private String wavExt = ".wav";
+	private String hplabExt = ".hplab";
+	protected int percent = 0;
+	private HTSEngine hmm_tts;
+	private HMMData htsData;
 
-    public boolean compute() throws Exception {
-        
-        // output mfcc dir creator 
-        File hmmParamDir =  new File(getProp(OUTHMMDIR));
-        if(!hmmParamDir.exists()){
-            System.out.print(hmmParamDir.getAbsolutePath()+" does not exist; ");
-            if (!hmmParamDir.mkdir()){
-                throw new Error("Could not create "+hmmParamDir.getAbsolutePath());
-            }
-            System.out.print("Created successfully.\n");
-        }
-        
-        
-        /* For initialise provide the name of the hmm voice and the name of its configuration file,
-         * also indicate the name of your MARY_BASE directory.*/
-        String MaryBase = getProp(MARYBASE);
-        //String locale = getProp(VOICECONFIG);
-        String voice = getProp(VOICENAME);
-        String configFile = getProp(VOICECONFIG);
-        // directory where the context features of each file are
-        String contextFeaDir = getProp(PHONEFEATS);
-        // the output dir has to be created already
-        String outputDir = getProp(OUTHMMDIR); 
-        // Create a htsengine object
-        hmm_tts = new HTSEngine();
-        
-        // Create and set HMMData
-        htsData = new HMMData();
-        htsData.initHMMData(voice, MaryBase, configFile);
-           
-        // Settings for using GV
-        if(getProp(USEGV).equals("true")){
-            htsData.setUseGV(true);
-        }
-        else{
-            htsData.setUseGV(false);
-        }
-        
-        //Settings for mixed excitation
-        htsData.setUseMixExc(true);
-        
-        // Now process all files, one by one
-        for (int i=0; i<bnl.getLength(); i++) {
-            percent = 100*i/bnl.getLength();
-            generateParameters(bnl.getName(i), contextFeaDir, outputDir);
-            boolean success = createHalfPhoneLab(bnl.getName(i));
-            if(!success) return false;
-        }
-        
-        return true;
-    }
+	protected void setupHelp() {
+		props2Help = new TreeMap();
+		props2Help.put(MARYBASE, "Mary Base.  ex: /home/user/MARY400/ ");
+		props2Help.put(VOICECONFIG, "locale ex: english");
+		props2Help.put(VOICENAME, "Voice name ex: slt-arctic");
+		props2Help.put(PHONEFEATS, "Phonefeatures directory");
+		props2Help.put(OUTHMMDIR, "Output directory to store hmm generated parameters");
+		props2Help.put(PPARAMETERS, "Generate parameters like MFCC, PITCH files?");
+		props2Help.put(PLAB, "Generate HMM Label files?");
+		props2Help.put(PWAVE, "Generate HMM WAVE files?");
+		props2Help.put(PSLAB, "Generate HMM State Label files?");
+		props2Help.put(USEGV, "Use Global variance(GV) in parameter generation?");
+	}
 
-    public int getProgress() {
-        return percent;
-    }
+	public SortedMap<String, String> getDefaultProps(DatabaseLayout db) {
+		this.db = db;
+		if (props == null) {
+			props = new TreeMap();
+			props.put(MARYBASE, db.getProp(db.MARYBASE));
+			props.put(VOICECONFIG, "english-hsmm-slt.config");
+			props.put(VOICENAME, "hsmm-slt");
+			props.put(PHONEFEATS, db.getProp(db.ROOTDIR) + File.separator + "phonefeatures");
+			props.put(OUTHMMDIR, db.getProp(db.ROOTDIR) + File.separator + "sCost" + File.separator + "hmmparams");
+			props.put(PPARAMETERS, "true");
+			props.put(PLAB, "true");
+			props.put(PWAVE, "false");
+			props.put(PSLAB, "false");
+			props.put(USEGV, "false");
+		}
+		return props;
+	}
+
+	public final String getName() {
+		return "HMMParameterExtractor";
+	}
+
+	@Override
+	protected void initialiseComp() {
+		// sCost dir creation, if doesn't exists
+		sCostDirectory = db.getProp(db.ROOTDIR) + File.separator + "sCost";
+		File sCostDir = new File(sCostDirectory);
+		if (!sCostDir.exists()) {
+			System.out.print(sCostDir.getAbsolutePath() + " does not exist; ");
+			if (!sCostDir.mkdir()) {
+				throw new Error("Could not create " + sCostDir.getAbsolutePath());
+			}
+			System.out.print("Created successfully.\n");
+		}
+	}
+
+	public boolean compute() throws Exception {
+
+		// output mfcc dir creator
+		File hmmParamDir = new File(getProp(OUTHMMDIR));
+		if (!hmmParamDir.exists()) {
+			System.out.print(hmmParamDir.getAbsolutePath() + " does not exist; ");
+			if (!hmmParamDir.mkdir()) {
+				throw new Error("Could not create " + hmmParamDir.getAbsolutePath());
+			}
+			System.out.print("Created successfully.\n");
+		}
+
+		/*
+		 * For initialise provide the name of the hmm voice and the name of its configuration file, also indicate the name of your
+		 * MARY_BASE directory.
+		 */
+		String MaryBase = getProp(MARYBASE);
+		// String locale = getProp(VOICECONFIG);
+		String voice = getProp(VOICENAME);
+		String configFile = getProp(VOICECONFIG);
+		// directory where the context features of each file are
+		String contextFeaDir = getProp(PHONEFEATS);
+		// the output dir has to be created already
+		String outputDir = getProp(OUTHMMDIR);
+		// Create a htsengine object
+		hmm_tts = new HTSEngine();
+
+		// Create and set HMMData
+		htsData = new HMMData();
+		htsData.initHMMData(voice, MaryBase, configFile);
+
+		// Settings for using GV
+		if (getProp(USEGV).equals("true")) {
+			htsData.setUseGV(true);
+		} else {
+			htsData.setUseGV(false);
+		}
+
+		// Settings for mixed excitation
+		htsData.setUseMixExc(true);
+
+		// Now process all files, one by one
+		for (int i = 0; i < bnl.getLength(); i++) {
+			percent = 100 * i / bnl.getLength();
+			generateParameters(bnl.getName(i), contextFeaDir, outputDir);
+			boolean success = createHalfPhoneLab(bnl.getName(i));
+			if (!success)
+				return false;
+		}
+
+		return true;
+	}
+
+	public int getProgress() {
+		return percent;
+	}
+
+	/**
+	 * Stand alone testing using a TARGETFEATURES list of files as input.
+	 * 
+	 * @param args
+	 * @throws IOException
+	 */
+	public void generateParameters(String file, String contextFeaDir, String outputDir) throws IOException, InterruptedException {
+
+		float fperiodmillisec = ((float) htsData.getFperiod() / (float) htsData.getRate()) * 1000;
+		float fperiodsec = ((float) htsData.getFperiod() / (float) htsData.getRate());
+		/* generate files out of HMMs */
+		String feaFile, parFile, durStateFile, durFile, mgcModifiedFile, outWavFile;
+		try {
+
+			/*
+			 * The utterance model, um, is a Vector (or linked list) of Model objects. It will contain the list of models for the
+			 * current label file.
+			 */
+			HTSUttModel um = new HTSUttModel();
+			HTSParameterGeneration pdf2par = new HTSParameterGeneration();
+			HTSVocoder par2speech = new HTSVocoder();
+			AudioInputStream ais;
+
+			/* Process label file of Mary context features and creates UttModel um. */
+			feaFile = contextFeaDir + file + pfeatExt;
+			um = hmm_tts.processUttFromFile(feaFile, htsData);
+
+			if (getProp(PLAB).equals("true")) {
+				/* save realised durations in a lab file */
+				FileWriter outputStream;
+				durFile = outputDir + file + labExt; /* realised durations */
+				outputStream = new FileWriter(durFile);
+				outputStream.write(hmm_tts.getRealisedDurations());
+				outputStream.close();
+			}
+			if (getProp(PSLAB).equals("true")) {
+				/* save realised durations at state label in a slab file */
+				float totalDur = 0;
+				int numStates = htsData.getCartTreeSet().getNumStates();
+				durStateFile = outputDir + file + slabExt; /* state level realised durations */
+				FileWriter outputStream = new FileWriter(durStateFile);
+				outputStream.write("#\n");
+				for (int i = 0; i < um.getNumModel(); i++) {
+					for (int j = 0; j < numStates; j++) {
+						totalDur += (um.getUttModel(i).getDur(j) * fperiodsec);
+						if (j < (numStates - 1))
+							outputStream.write(totalDur + " 0 " + um.getUttModel(i).getPhoneName() + "\n");
+						else
+							outputStream.write(totalDur + " 1 " + um.getUttModel(i).getPhoneName() + "\n");
+					}
+				}
+				outputStream.close();
+			}
+
+			if (getProp(PPARAMETERS).equals("true")) {
+				/* Generate sequence of speech parameter vectors, generate parameters out of sequence of pdf's */
+				boolean debug = true; /*
+									 * with debug=true it saves the generated parameters f0 and mfcc in parFile.f0 and
+									 * parFile.mfcc in Mary format.
+									 */
+				parFile = outputDir + file; /* generated parameters mfcc and f0, Mary format */
+				pdf2par.htsMaximumLikelihoodParameterGeneration(um, htsData);
+			}
+
+			if (getProp(PWAVE).equals("true")) {
+				/* Synthesize speech waveform, generate speech out of sequence of parameter */
+				ais = par2speech.htsMLSAVocoder(pdf2par, htsData);
+				outWavFile = outputDir + file + wavExt; /* generated wav file */
+				System.out.println("saving to file: " + outWavFile);
+				File fileOut = new File(outWavFile);
+				if (AudioSystem.isFileTypeSupported(AudioFileFormat.Type.WAVE, ais)) {
+					AudioSystem.write(ais, AudioFileFormat.Type.WAVE, fileOut);
+				}
+			}
+		} catch (Exception e) {
+			System.err.println("Exception: " + e.getMessage());
+		}
+
+	} /* main method */
+
+	private boolean createHalfPhoneLab(String baseName) {
+		String hmmDir = getProp(OUTHMMDIR);
+		if (!getProp(PLAB).equals("true"))
+			return true;
+
+		String labFile = hmmDir + File.separator + baseName + labExt;
+		try {
+			UnitLabel[] unitLab = UnitLabel.readLabFile(labFile);
+			PrintWriter pw = new PrintWriter(new FileWriter(new File(hmmDir + File.separator + baseName + hplabExt)));
+			pw.println("#");
+			int unitIndex = 0;
+			for (int i = 0; i < unitLab.length; i++) {
+				double duration = unitLab[i].endTime - unitLab[i].startTime;
+				assert duration > 0 : "Duration is not > 0 for phone " + unitLab[i].unitName + " (" + baseName + ")";
+				double midTime = unitLab[i].startTime + duration / 2;
+				unitIndex++;
+				String leftUnit = midTime + " " + unitIndex + " " + unitLab[i].unitName + "_L";
+				unitIndex++;
+				String rightUnit = unitLab[i].endTime + " " + unitIndex + " " + unitLab[i].unitName + "_R";
+				pw.println(leftUnit);
+				pw.println(rightUnit);
+			}
+			pw.flush();
+			pw.close();
+		} catch (IOException e) {
+			return false;
+		}
+		return true;
+	}
 
-    /** 
-     * Stand alone testing using a TARGETFEATURES list of files as input. 
-     * @param args
-     * @throws IOException
-     */
-    public void generateParameters(String file, String contextFeaDir, String outputDir) throws IOException, InterruptedException{
-       
-        float fperiodmillisec = ((float)htsData.getFperiod() / (float)htsData.getRate()) * 1000;
-        float fperiodsec = ((float)htsData.getFperiod() / (float)htsData.getRate());
-        /* generate files out of HMMs */
-        String feaFile, parFile, durStateFile, durFile, mgcModifiedFile, outWavFile;
-        try {
-          
-          /* The utterance model, um, is a Vector (or linked list) of Model objects. 
-           * It will contain the list of models for the current label file. */
-          HTSUttModel um = new HTSUttModel();
-          HTSParameterGeneration pdf2par = new HTSParameterGeneration();
-          HTSVocoder par2speech = new HTSVocoder();
-          AudioInputStream ais;
-          
-          
-          /* Process label file of Mary context features and creates UttModel um.   */
-          feaFile      = contextFeaDir + file + pfeatExt;
-          um = hmm_tts.processUttFromFile(feaFile, htsData);
-            
-          
-          if(getProp(PLAB).equals("true")){
-              /* save realised durations in a lab file */
-              FileWriter outputStream;
-              durFile      = outputDir + file + labExt;   /* realised durations */
-              outputStream = new FileWriter(durFile);
-              outputStream.write(hmm_tts.getRealisedDurations());
-              outputStream.close();
-          }
-          if(getProp(PSLAB).equals("true")){
-              /* save realised durations at state label in a slab file */
-              float totalDur=0;
-              int numStates=htsData.getCartTreeSet().getNumStates();
-              durStateFile = outputDir + file + slabExt;  /* state level realised durations */
-              FileWriter outputStream = new FileWriter(durStateFile);
-              outputStream.write("#\n");
-              for(int i=0; i<um.getNumModel(); i++){
-                for(int j=0; j<numStates; j++){
-                    totalDur += (um.getUttModel(i).getDur(j)*fperiodsec);
-                    if(j < (numStates-1))
-                      outputStream.write(totalDur + " 0 " + um.getUttModel(i).getPhoneName() + "\n");
-                    else
-                      outputStream.write(totalDur + " 1 " + um.getUttModel(i).getPhoneName() + "\n");   
-                  }         
-              }
-              outputStream.close();
-          }
-          
-          if(getProp(PPARAMETERS).equals("true")){
-              /* Generate sequence of speech parameter vectors, generate parameters out of sequence of pdf's */ 
-              boolean debug = true;  /* with debug=true it saves the generated parameters  
-                                        f0 and mfcc in parFile.f0 and parFile.mfcc in Mary format. */
-              parFile      = outputDir + file;            /* generated parameters mfcc and f0, Mary format */
-              pdf2par.htsMaximumLikelihoodParameterGeneration(um, htsData);
-          }
-          
-          if(getProp(PWAVE).equals("true")){
-              /* Synthesize speech waveform, generate speech out of sequence of parameter */
-              ais = par2speech.htsMLSAVocoder(pdf2par, htsData);
-              outWavFile   = outputDir + file + wavExt;   /* generated wav file */
-              System.out.println("saving to file: " + outWavFile);
-              File fileOut = new File(outWavFile);
-              if (AudioSystem.isFileTypeSupported(AudioFileFormat.Type.WAVE,ais)) {
-                AudioSystem.write(ais, AudioFileFormat.Type.WAVE, fileOut);
-              }
-          }
-        } catch (Exception e) {
-            System.err.println("Exception: " + e.getMessage());
-        }
-      
-    }  /* main method */
-    
-    
-    private boolean createHalfPhoneLab(String baseName){
-        String hmmDir = getProp(OUTHMMDIR);
-        if(!getProp(PLAB).equals("true")) return true;
-        
-        String labFile = hmmDir+File.separator+baseName+labExt;
-        try {
-            UnitLabel[] unitLab = UnitLabel.readLabFile(labFile);
-            PrintWriter pw = new PrintWriter(new FileWriter(new File(hmmDir+File.separator+baseName+hplabExt)));
-            pw.println("#");
-            int unitIndex = 0;
-            for(int i=0; i<unitLab.length;i++){
-                double duration = unitLab[i].endTime - unitLab[i].startTime;
-                assert duration > 0 : "Duration is not > 0 for phone "+unitLab[i].unitName+" ("+baseName+")";
-                double midTime = unitLab[i].startTime + duration/2;
-                unitIndex++;
-                String leftUnit = midTime + " " + unitIndex + " " + unitLab[i].unitName + "_L";
-                unitIndex++;
-                String rightUnit = unitLab[i].endTime + " "+ unitIndex + " "+unitLab[i].unitName+"_R";
-                pw.println(leftUnit);
-                pw.println(rightUnit);
-            }
-            pw.flush();
-            pw.close();
-        } catch (IOException e) {
-            return false;
-        }
-        return true;
-    }
-    
 }
-

marytts-builder/src/main/java/marytts/tools/voiceimport/HMMVoiceCompiler.java

@@ -15,77 +15,73 @@ import java.util.StringTokenizer;
 
 import org.apache.commons.io.FileUtils;
 
-
 /**
  * @author marc, marcela
  *
  */
 public class HMMVoiceCompiler extends VoiceCompiler {
-	
-    // constants to access filenames in database component properties and organize file list:
-	
-    /** HMM Voice-specific parameters, these are parameters used during models training
-    if using MGC: 
-             gamma=0  alpha=0.42 linear gain (default)
-    if using LSP: gamma>0 
-        LSP: gamma=1  alpha=0.0  linear gain/log gain 
-    Mel-LSP: gamma=1  alpha=0.42 log gain
-    MGC-LSP: gamma=3  alpha=0.42 log gain  */
-    public static final String alpha = "HMMVoiceConfigure.freqWarp";
-    public static final String gamma = "HMMVoiceConfigure.gamma";    
-    public static final String logGain = "HMMVoiceConfigure.lnGain";    
-    
-    /** Sampling frequency and frame period have to be specified (sampling freq is included in the general config) */
-    public static final String samplingRate = "HMMVoiceConfigure.sampFreq";
-    public static final String framePeriod = "HMMVoiceConfigure.frameShift";
-           
-    /** The following files depend on the version number and question number defined during configuration and training */
-    public static final String questionNumber = "HMMVoiceConfigure.questionsNum";
-    public static final String versionNumber = "HMMVoiceConfigure.version";
-    
-    /** Tree files and TreeSet object */
-    public static String treeDurFile;
-    public static String treeLf0File;
-    public static String treeMcpFile;
-    public static String treeStrFile;
-        
-    /** HMM pdf model files and ModelSet object */
-    public static String pdfDurFile;
-    public static String pdfLf0File;
-    public static String pdfMcpFile;
-    public static String pdfStrFile;
 
-    /** Global variance files */
-    public static String pdfLf0GvFile; 
-    public static String pdfMcpGvFile;  
-    public static String pdfStrGvFile;
-            
-    /** Variables for mixed excitation */
-    public static final String mixFiltersFile = "HMMVoiceConfigure.strFilterFileName";
-    public static String mixFiltersFileLocation;
-    public static final String numFilters = "HMMVoiceConfigure.strOrder";
-    
-   /** Example context feature file (TARGETFEATURES in MARY) */
-    public static String featuresFileExample = "mary/features_example.pfeats";
-    
-    public String FeaFile;
-    /** trickyPhones file if any, this file could have been created during makeQuestions and makeLabels
-     * if it was created, because there are tricky phones in the allophones set, then it should be in
-     * voiceDIR/mary/trickyPhones.txt */
-    public static final String trickyPhonesFile = "HMMVoiceMakeData.trickyPhonesFile";
-    public static final String hmmFeaturesMapFile = "HMMVoiceMakeData.featureListMapFile";
-    
-    /** Mapping in case of using alias names for extra features during training */
-    Map<String,String> actualFeatureNames = new HashMap<String, String>();
+	// constants to access filenames in database component properties and organize file list:
+
+	/**
+	 * HMM Voice-specific parameters, these are parameters used during models training if using MGC: gamma=0 alpha=0.42 linear
+	 * gain (default) if using LSP: gamma>0 LSP: gamma=1 alpha=0.0 linear gain/log gain Mel-LSP: gamma=1 alpha=0.42 log gain
+	 * MGC-LSP: gamma=3 alpha=0.42 log gain
+	 */
+	public static final String alpha = "HMMVoiceConfigure.freqWarp";
+	public static final String gamma = "HMMVoiceConfigure.gamma";
+	public static final String logGain = "HMMVoiceConfigure.lnGain";
+
+	/** Sampling frequency and frame period have to be specified (sampling freq is included in the general config) */
+	public static final String samplingRate = "HMMVoiceConfigure.sampFreq";
+	public static final String framePeriod = "HMMVoiceConfigure.frameShift";
+
+	/** The following files depend on the version number and question number defined during configuration and training */
+	public static final String questionNumber = "HMMVoiceConfigure.questionsNum";
+	public static final String versionNumber = "HMMVoiceConfigure.version";
+
+	/** Tree files and TreeSet object */
+	public static String treeDurFile;
+	public static String treeLf0File;
+	public static String treeMcpFile;
+	public static String treeStrFile;
+
+	/** HMM pdf model files and ModelSet object */
+	public static String pdfDurFile;
+	public static String pdfLf0File;
+	public static String pdfMcpFile;
+	public static String pdfStrFile;
+
+	/** Global variance files */
+	public static String pdfLf0GvFile;
+	public static String pdfMcpGvFile;
+	public static String pdfStrGvFile;
+
+	/** Variables for mixed excitation */
+	public static final String mixFiltersFile = "HMMVoiceConfigure.strFilterFileName";
+	public static String mixFiltersFileLocation;
+	public static final String numFilters = "HMMVoiceConfigure.strOrder";
+
+	/** Example context feature file (TARGETFEATURES in MARY) */
+	public static String featuresFileExample = "mary/features_example.pfeats";
+
+	public String FeaFile;
+	/**
+	 * trickyPhones file if any, this file could have been created during makeQuestions and makeLabels if it was created, because
+	 * there are tricky phones in the allophones set, then it should be in voiceDIR/mary/trickyPhones.txt
+	 */
+	public static final String trickyPhonesFile = "HMMVoiceMakeData.trickyPhonesFile";
+	public static final String hmmFeaturesMapFile = "HMMVoiceMakeData.featureListMapFile";
+
+	/** Mapping in case of using alias names for extra features during training */
+	Map<String, String> actualFeatureNames = new HashMap<String, String>();
 
-	
 	/**
 	 * 
 	 */
 	public HMMVoiceCompiler() {
 	}
 
-
 	/**
 	 * @throws IOException
 	 * @throws FileNotFoundException
@@ -94,72 +90,69 @@ public class HMMVoiceCompiler extends VoiceCompiler {
 	protected void mapFeatures() throws IOException, FileNotFoundException {
 		String rootDir = db.getProp(DatabaseLayout.ROOTDIR);
 		// First find a features file example
-		getFeatureFileExample();	
-		
+		getFeatureFileExample();
+
 		/* Substitute question number and version number */
 		String vnum = db.getProperty(versionNumber);
-	    String qnum = db.getProperty(questionNumber);
-	       
-	    /** Tree files and TreeSet object */
-	    treeDurFile = "hts/voices/qst" + qnum + "/ver" + vnum + "/tree-dur.inf";
-	    treeLf0File = "hts/voices/qst" + qnum + "/ver" + vnum + "/tree-lf0.inf";
-	    treeMcpFile = "hts/voices/qst" + qnum + "/ver" + vnum + "/tree-mgc.inf";
-	    treeStrFile = "hts/voices/qst" + qnum + "/ver" + vnum + "/tree-str.inf";
-	        
-	    /** HMM pdf model files and ModelSet object */
-	    pdfDurFile = "hts/voices/qst" + qnum + "/ver" + vnum + "/dur.pdf";
-	    pdfLf0File = "hts/voices/qst" + qnum + "/ver" + vnum + "/lf0.pdf";
-	    pdfMcpFile = "hts/voices/qst" + qnum + "/ver" + vnum + "/mgc.pdf";
-	    pdfStrFile = "hts/voices/qst" + qnum + "/ver" + vnum + "/str.pdf";
-	    
-	    /** Global variance files */
-	    pdfLf0GvFile = "hts/voices/qst" + qnum + "/ver" + vnum + "/gv-lf0.pdf"; 
-	    pdfMcpGvFile = "hts/voices/qst" + qnum + "/ver" + vnum + "/gv-mgc.pdf";  
-	    pdfStrGvFile = "hts/voices/qst" + qnum + "/ver" + vnum + "/gv-str.pdf";
-	    
-	    /** Filter file for mixed excitation */
-	    mixFiltersFileLocation = "hts/data/" + db.getProperty(mixFiltersFile);
-	    
-	    // Set files for resources
-	    // Now I know the names of the resources so I can set the files for resources on the maven compiler
-	    String[] filenamesResources = new String[] {
-			    rootDir+treeDurFile, rootDir+treeLf0File, rootDir+treeMcpFile, rootDir+treeStrFile, 
-			    rootDir+pdfDurFile, rootDir+pdfLf0File, rootDir+pdfMcpFile, rootDir+pdfStrFile, 
-			    rootDir+pdfLf0GvFile, rootDir+pdfMcpGvFile, rootDir+pdfStrGvFile, 
-			    rootDir+mixFiltersFileLocation, rootDir+featuresFileExample, 
-			    rootDir+db.getProperty(trickyPhonesFile) 
-		};
+		String qnum = db.getProperty(questionNumber);
+
+		/** Tree files and TreeSet object */
+		treeDurFile = "hts/voices/qst" + qnum + "/ver" + vnum + "/tree-dur.inf";
+		treeLf0File = "hts/voices/qst" + qnum + "/ver" + vnum + "/tree-lf0.inf";
+		treeMcpFile = "hts/voices/qst" + qnum + "/ver" + vnum + "/tree-mgc.inf";
+		treeStrFile = "hts/voices/qst" + qnum + "/ver" + vnum + "/tree-str.inf";
+
+		/** HMM pdf model files and ModelSet object */
+		pdfDurFile = "hts/voices/qst" + qnum + "/ver" + vnum + "/dur.pdf";
+		pdfLf0File = "hts/voices/qst" + qnum + "/ver" + vnum + "/lf0.pdf";
+		pdfMcpFile = "hts/voices/qst" + qnum + "/ver" + vnum + "/mgc.pdf";
+		pdfStrFile = "hts/voices/qst" + qnum + "/ver" + vnum + "/str.pdf";
+
+		/** Global variance files */
+		pdfLf0GvFile = "hts/voices/qst" + qnum + "/ver" + vnum + "/gv-lf0.pdf";
+		pdfMcpGvFile = "hts/voices/qst" + qnum + "/ver" + vnum + "/gv-mgc.pdf";
+		pdfStrGvFile = "hts/voices/qst" + qnum + "/ver" + vnum + "/gv-str.pdf";
+
+		/** Filter file for mixed excitation */
+		mixFiltersFileLocation = "hts/data/" + db.getProperty(mixFiltersFile);
+
+		// Set files for resources
+		// Now I know the names of the resources so I can set the files for resources on the maven compiler
+		String[] filenamesResources = new String[] { rootDir + treeDurFile, rootDir + treeLf0File, rootDir + treeMcpFile,
+				rootDir + treeStrFile, rootDir + pdfDurFile, rootDir + pdfLf0File, rootDir + pdfMcpFile, rootDir + pdfStrFile,
+				rootDir + pdfLf0GvFile, rootDir + pdfMcpGvFile, rootDir + pdfStrGvFile, rootDir + mixFiltersFileLocation,
+				rootDir + featuresFileExample, rootDir + db.getProperty(trickyPhonesFile) };
 		File[] filesForResources = new File[filenamesResources.length];
-		for (int i=0; i<filenamesResources.length; i++) {
+		for (int i = 0; i < filenamesResources.length; i++) {
 			filesForResources[i] = new File(filenamesResources[i]);
-		}		
+		}
 		compiler.setFilesForResources(filesForResources);
-		
-		
-        // Before setting the tree files, we need to check if they contain aliases for the extra features used for training
-        // if so there must be a file mary/hmmFeaturesMap.txt which has to be used to convert back the feature names
-        // Check if features map was used		
-		String feasMapFileName = rootDir + db.getProperty(hmmFeaturesMapFile); 
-        System.out.println("Checking if aliases for extra features used for training were used: checking if file exist -->" + feasMapFileName);
-        File featuresMap = new File(feasMapFileName);
-        
-        if(featuresMap.exists()) {
-          // convert back the features in all tree files: treeDurFile, treeLf0File, treeMcpFile, treeStrFile
-          System.out.println("convert back the features in all tree files: treeDurFile, treeLf0File, treeMcpFile, treeStrFile");	          
-          loadFeaturesMap(feasMapFileName);	
-          
-          replaceBackFeatureNames(rootDir + treeDurFile);
-          replaceBackFeatureNames(rootDir + treeLf0File);
-          replaceBackFeatureNames(rootDir + treeMcpFile);
-          replaceBackFeatureNames(rootDir + treeStrFile);
-        }
+
+		// Before setting the tree files, we need to check if they contain aliases for the extra features used for training
+		// if so there must be a file mary/hmmFeaturesMap.txt which has to be used to convert back the feature names
+		// Check if features map was used
+		String feasMapFileName = rootDir + db.getProperty(hmmFeaturesMapFile);
+		System.out.println("Checking if aliases for extra features used for training were used: checking if file exist -->"
+				+ feasMapFileName);
+		File featuresMap = new File(feasMapFileName);
+
+		if (featuresMap.exists()) {
+			// convert back the features in all tree files: treeDurFile, treeLf0File, treeMcpFile, treeStrFile
+			System.out.println("convert back the features in all tree files: treeDurFile, treeLf0File, treeMcpFile, treeStrFile");
+			loadFeaturesMap(feasMapFileName);
+
+			replaceBackFeatureNames(rootDir + treeDurFile);
+			replaceBackFeatureNames(rootDir + treeLf0File);
+			replaceBackFeatureNames(rootDir + treeMcpFile);
+			replaceBackFeatureNames(rootDir + treeStrFile);
+		}
 	}
 
 	@Override
 	protected boolean isUnitSelectionVoice() {
 		return false;
 	}
-	
+
 	@Override
 	protected String getCompileDirProp() {
 		return "HMMVoiceCompiler.compileDir";
@@ -167,14 +160,12 @@ public class HMMVoiceCompiler extends VoiceCompiler {
 
 	@Override
 	protected String getVoiceName(DatabaseLayout db) {
-		return db.getVoiceName()+"-hsmm";
+		return db.getVoiceName() + "-hsmm";
 	}
 
-
-
-
-
-	/* (non-Javadoc)
+	/*
+	 * (non-Javadoc)
+	 * 
 	 * @see marytts.tools.voiceimport.VoiceImportComponent#getName()
 	 */
 	@Override
@@ -186,181 +177,181 @@ public class HMMVoiceCompiler extends VoiceCompiler {
 	protected Map<String, String> getExtraVariableSubstitutionMap() {
 		Map<String, String> m = new HashMap<String, String>();
 		m.put("FRAMEPERIOD", String.valueOf(db.getProperty(framePeriod)));
-		
+
 		m.put("ALPHA", String.valueOf(db.getProperty(alpha)));
 		m.put("GAMMA", String.valueOf(db.getProperty(gamma)));
 		m.put("LOGGAIN", String.valueOf(db.getProperty(logGain)));
-				
+
 		// The filters file name includes the "filters/" directory, we need here just the file name
 		m.put("MIXEXCFILTERFILE", db.getProperty(mixFiltersFile).substring(8));
-		m.put("NUMMIXEXCFILTERS", String.valueOf(db.getProperty(numFilters)));		
-				
+		m.put("NUMMIXEXCFILTERS", String.valueOf(db.getProperty(numFilters)));
+
 		return m;
 	}
 
 	@Override
 	protected File[] getFilesForResources() {
-		// at this point, initialisation, I still do not know where 
+		// at this point, initialisation, I still do not know where
 		// exactly the files will be
-		/*		
-		String[] filenamesResources = new String[] {
-			    treeDurFile, treeLf0File, treeMcpFile, treeStrFile, pdfDurFile, pdfLf0File, pdfMcpFile, pdfStrFile, 
-			    pdfLf0GvFile, pdfMcpGvFile, pdfStrGvFile, mixFiltersFileLocation, featuresFileExample, 
-			    db.getProperty(trickyPhonesFile) 
-		};
-		File[] filesForResources = new File[filenamesResources.length];
-		for (int i=0; i<filenamesResources.length; i++) {
-			filesForResources[i] = new File(filenamesResources[i]);
-		}
-		return filesForResources;
-		*/
+		/*
+		 * String[] filenamesResources = new String[] { treeDurFile, treeLf0File, treeMcpFile, treeStrFile, pdfDurFile,
+		 * pdfLf0File, pdfMcpFile, pdfStrFile, pdfLf0GvFile, pdfMcpGvFile, pdfStrGvFile, mixFiltersFileLocation,
+		 * featuresFileExample, db.getProperty(trickyPhonesFile) }; File[] filesForResources = new
+		 * File[filenamesResources.length]; for (int i=0; i<filenamesResources.length; i++) { filesForResources[i] = new
+		 * File(filenamesResources[i]); } return filesForResources;
+		 */
 		return null;
 	}
-	
+
 	@Override
 	protected File[] getFilesForFilesystem() {
 		return null;
 	}
 
-	
-	protected void getFeatureFileExample() throws IOException {	
-		String fileExample=null;
+	protected void getFeatureFileExample() throws IOException {
+		String fileExample = null;
 		String rootDir = db.getProp(DatabaseLayout.ROOTDIR);
-		logger.info("Getting a context feature file example in phonefeatures/");	
-		
-        /* copy one example of MARY context features file, it can be one of the 
-         * files used for testing in phonefeatures/*.pfeats*/	
-        File dirPhonefeatures  = new File(rootDir + "phonefeatures/");
-        if( dirPhonefeatures.exists() && dirPhonefeatures.list().length > 0 ){ 
-          String[] feaFiles = dirPhonefeatures.list();
-          fileExample = feaFiles[0];          
-          File in = new File(rootDir + "phonefeatures/" + fileExample);
-          
-          if(in.isDirectory() ) {
-        	logger.info("HMMVoiceConfigure.adaptScripts = " + db.getProperty("HMMVoiceConfigure.adaptScripts"));
-            /* If adaptive training then look for an example in the first directory */        	
-        	if( in.exists() && in.list().length > 0 ){                  
-              FileUtils.copyFile(new File(rootDir, "phonefeatures/" + fileExample + "/" + in.list()[0]), new File(rootDir, featuresFileExample));
-            }  
-          } else if(in.exists()){
-        	  FileUtils.copyFile(in, new File(rootDir, featuresFileExample));
-          } else {
-        	System.out.println("Problem copying one example of context features, the directory phonefeatures/ is empty(?)");
-            throw new IOException();  
-          }
-          
-        } else{
-           System.out.println("Problem copying one example of context features, the directory does not exist.");
-           throw new IOException();
-        }         
+		logger.info("Getting a context feature file example in phonefeatures/");
+
+		/*
+		 * copy one example of MARY context features file, it can be one of the files used for testing in phonefeatures/*.pfeats
+		 */
+		File dirPhonefeatures = new File(rootDir + "phonefeatures/");
+		if (dirPhonefeatures.exists() && dirPhonefeatures.list().length > 0) {
+			String[] feaFiles = dirPhonefeatures.list();
+			fileExample = feaFiles[0];
+			File in = new File(rootDir + "phonefeatures/" + fileExample);
+
+			if (in.isDirectory()) {
+				logger.info("HMMVoiceConfigure.adaptScripts = " + db.getProperty("HMMVoiceConfigure.adaptScripts"));
+				/* If adaptive training then look for an example in the first directory */
+				if (in.exists() && in.list().length > 0) {
+					FileUtils.copyFile(new File(rootDir, "phonefeatures/" + fileExample + "/" + in.list()[0]), new File(rootDir,
+							featuresFileExample));
+				}
+			} else if (in.exists()) {
+				FileUtils.copyFile(in, new File(rootDir, featuresFileExample));
+			} else {
+				System.out.println("Problem copying one example of context features, the directory phonefeatures/ is empty(?)");
+				throw new IOException();
+			}
+
+		} else {
+			System.out.println("Problem copying one example of context features, the directory does not exist.");
+			throw new IOException();
+		}
 	}
-	
-    /**
-     * Replace the aliases for features used during training
-     * @param treeFileName a HTS tree file
-     * @throws IOException
-     */
-    private void replaceBackFeatureNames(String treeFileName) throws IOException{
-      
-      BufferedReader s = null; 
-      FileWriter outputStream;
-      
-      //---outputStream.write(hmm_tts.getRealisedDurations());
-      //---outputStream.close();
-      String line;
-      // read the file until the symbol the delimits an state is found
-      try {   
-        // output file to copy the result
-        outputStream = new FileWriter(treeFileName + ".tmp");
-        
-        // read lines of tree-*.inf fileName 
-        s = new BufferedReader(new InputStreamReader(new FileInputStream(treeFileName)));
-        logger.info("load: reading " + treeFileName);
-            
-        // skip questions section, but copy the lines on the temporary output
-        while((line = s.readLine()) != null) {
-          outputStream.write(line + "\n");
-          if (line.indexOf("QS") < 0 ) break;   /* a new state is indicated by {*}[2], {*}[3], ... */
-        }
-        
-        StringTokenizer sline;
-        String buf1, buf2, buf3, buf4;
-        while((line = s.readLine()) != null) {
-          //System.out.println("line: " + line);
-          if(line.indexOf("{") >= 0 || line.indexOf("}") >= 0 || line.length()==0){ /* this is the indicator of a new state-tree */
-            outputStream.write(line + "\n");           
-          } else {              
-            sline = new StringTokenizer(line);
-            buf1 = sline.nextToken();
-            buf2 = sline.nextToken();            
-            String [] fea = buf2.split("=");            
-            buf3 = sline.nextToken();            
-            buf4 = sline.nextToken();
-            //System.out.format("newLine:  %s %s=%s\t\t%s\t%s\n", buf1, replaceBack(fea[0]), fea[1], buf3, buf4);            
-            outputStream.write(" " + buf1 + " " + replaceBack(fea[0]) + "=" + fea[1] + "\t\t" + buf3 + "\t" + buf4 + "\n");            
-          }
-        }
-        outputStream.close();
-        System.out.println("Features alises replaced in file: " + treeFileName + ".tmp");  
-        // now replace the file
-        FileUtils.copyFile(new File(treeFileName + ".tmp"), new File(treeFileName)); 
-        System.out.println("Copied file: " + treeFileName + ".tmp" + "  to: " + treeFileName + "\n"); 
-        
-       } catch (IOException e) {
-              logger.debug("FileNotFoundException: " + e.getMessage());
-              throw new IOException("LoadTreeSet: ", e);
-      }
-   
-    }
-    
-    /**
-     * Load mapping of features from file
-     * @param fileName 
-     * @throws FileNotFoundException
-     */
-    private Map<String,String> loadFeaturesMap(String fileName) throws FileNotFoundException{
-        
-        Scanner aliasList = null;
-        try {
-          aliasList = new Scanner(new BufferedReader(new FileReader(fileName)));
-          String line;
-          logger.info("loading features map from file: " + fileName);
-          while (aliasList.hasNext()) {
-            line = aliasList.nextLine();
-            String[] fea = line.split(" ");
-                      
-            actualFeatureNames.put(fea[1], fea[0]);
-            logger.info("  " + fea[0] + " -->  " + fea[1]);             
-          }
-          if (aliasList != null) { 
-            aliasList.close();
-          }       
-        } catch (FileNotFoundException e) {
-            logger.debug("loadTrickyPhones:  " + e.getMessage());
-             throw e;
-        } 
-        return actualFeatureNames;
-    }
- 
 
-    
-    /**
-     * Replace label with information in the global map list actualFeatureNames
-     * @param lab replaced label
-     * @return
-     */
-    public String replaceBack(String lab){
-      
-      String s = lab;
-        
-      if( actualFeatureNames.containsKey(lab)){
-         s = actualFeatureNames.get(lab); 
-      } 
-      return s;        
-    }
-    
-    
+	/**
+	 * Replace the aliases for features used during training
+	 * 
+	 * @param treeFileName
+	 *            a HTS tree file
+	 * @throws IOException
+	 */
+	private void replaceBackFeatureNames(String treeFileName) throws IOException {
+
+		BufferedReader s = null;
+		FileWriter outputStream;
+
+		// ---outputStream.write(hmm_tts.getRealisedDurations());
+		// ---outputStream.close();
+		String line;
+		// read the file until the symbol the delimits an state is found
+		try {
+			// output file to copy the result
+			outputStream = new FileWriter(treeFileName + ".tmp");
+
+			// read lines of tree-*.inf fileName
+			s = new BufferedReader(new InputStreamReader(new FileInputStream(treeFileName)));
+			logger.info("load: reading " + treeFileName);
+
+			// skip questions section, but copy the lines on the temporary output
+			while ((line = s.readLine()) != null) {
+				outputStream.write(line + "\n");
+				if (line.indexOf("QS") < 0)
+					break; /* a new state is indicated by {*}[2], {*}[3], ... */
+			}
+
+			StringTokenizer sline;
+			String buf1, buf2, buf3, buf4;
+			while ((line = s.readLine()) != null) {
+				// System.out.println("line: " + line);
+				if (line.indexOf("{") >= 0 || line.indexOf("}") >= 0 || line.length() == 0) { /*
+																							 * this is the indicator of a new
+																							 * state-tree
+																							 */
+					outputStream.write(line + "\n");
+				} else {
+					sline = new StringTokenizer(line);
+					buf1 = sline.nextToken();
+					buf2 = sline.nextToken();
+					String[] fea = buf2.split("=");
+					buf3 = sline.nextToken();
+					buf4 = sline.nextToken();
+					// System.out.format("newLine:  %s %s=%s\t\t%s\t%s\n", buf1, replaceBack(fea[0]), fea[1], buf3, buf4);
+					outputStream
+							.write(" " + buf1 + " " + replaceBack(fea[0]) + "=" + fea[1] + "\t\t" + buf3 + "\t" + buf4 + "\n");
+				}
+			}
+			outputStream.close();
+			System.out.println("Features alises replaced in file: " + treeFileName + ".tmp");
+			// now replace the file
+			FileUtils.copyFile(new File(treeFileName + ".tmp"), new File(treeFileName));
+			System.out.println("Copied file: " + treeFileName + ".tmp" + "  to: " + treeFileName + "\n");
+
+		} catch (IOException e) {
+			logger.debug("FileNotFoundException: " + e.getMessage());
+			throw new IOException("LoadTreeSet: ", e);
+		}
+
+	}
+
+	/**
+	 * Load mapping of features from file
+	 * 
+	 * @param fileName
+	 * @throws FileNotFoundException
+	 */
+	private Map<String, String> loadFeaturesMap(String fileName) throws FileNotFoundException {
+
+		Scanner aliasList = null;
+		try {
+			aliasList = new Scanner(new BufferedReader(new FileReader(fileName)));
+			String line;
+			logger.info("loading features map from file: " + fileName);
+			while (aliasList.hasNext()) {
+				line = aliasList.nextLine();
+				String[] fea = line.split(" ");
+
+				actualFeatureNames.put(fea[1], fea[0]);
+				logger.info("  " + fea[0] + " -->  " + fea[1]);
+			}
+			if (aliasList != null) {
+				aliasList.close();
+			}
+		} catch (FileNotFoundException e) {
+			logger.debug("loadTrickyPhones:  " + e.getMessage());
+			throw e;
+		}
+		return actualFeatureNames;
+	}
+
+	/**
+	 * Replace label with information in the global map list actualFeatureNames
+	 * 
+	 * @param lab
+	 *            replaced label
+	 * @return
+	 */
+	public String replaceBack(String lab) {
+
+		String s = lab;
+
+		if (actualFeatureNames.containsKey(lab)) {
+			s = actualFeatureNames.get(lab);
+		}
+		return s;
+	}
 
-	
-	
 }

marytts-builder/src/main/java/marytts/tools/voiceimport/HMMVoiceConfigure.java

@@ -61,7 +61,6 @@
  *
  */
 
-
 package marytts.tools.voiceimport;
 
 import java.io.File;
@@ -70,378 +69,347 @@ import java.util.TreeMap;
 
 import marytts.util.io.General;
 
+public class HMMVoiceConfigure extends VoiceImportComponent {
 
+	private DatabaseLayout db;
+	private String name = "HMMVoiceConfigure";
+	public final String ADAPTSCRIPTS = name + ".adaptScripts";
 
-public class HMMVoiceConfigure extends VoiceImportComponent{
-    
-    private DatabaseLayout db;
-    private String name = "HMMVoiceConfigure";
-    public final String ADAPTSCRIPTS  = name+".adaptScripts";
-    
-    /** settings for speaker independent training */    
-    public final String CONFIGUREFILE = name+".configureFile";
-    public final String SPEAKER       = name+".speaker";
-    public final String DATASET       = name+".dataSet";
-    public final String LOWERF0       = name+".lowerF0";
-    public final String UPPERF0       = name+".upperF0";
-    public final String NUMTESTFILES  = name+".numTestFiles";
-    
-    public final String VER           = name+".version";
-    public final String QNUM          = name+".questionsNum";
-    public final String FRAMELEN      = name+".frameLen";
-    public final String FRAMESHIFT    = name+".frameShift";
-    public final String WINDOWTYPE    = name+".windowType";
-    public final String NORMALIZE     = name+".normalize";
-    public final String FFTLEN        = name+".fftLen";
-    public final String FREQWARP      = name+".freqWarp";
-    public final String GAMMA         = name+".gamma";
-    public final String MGCORDER      = name+".mgcOrder";
-    public final String STRORDER      = name+".strOrder";
-    public final String STRFILTERNAME = name+".strFilterFileName";
-    public final String MGCBANDWIDTH  = name+".mgcBandWidth";
-    public final String STRBANDWIDTH  = name+".strBandWidth";
-    public final String LF0BANDWIDTH  = name+".lf0BandWidth";
-    
-    public final String LNGAIN        = name+".lnGain";
-    public final String SAMPFREQ      = name+".sampfreq";
-    public final String NSTATE        = name+".numState";
-    public final String NITER         = name+".numIterations";
-    
-    /** settings for HTS ADAPT training scripts */
-    public final String ADAPTTRAINSPKR = name+".adaptTrainSpkr";
-    public final String ADAPTSPKR      = name+".adaptSpkr";
-    public final String ADAPTF0_RANGES = name+".adaptF0Ranges";
-    public final String ADAPTSPKRMASK  = name+".adaptSpkrMask";
-    public final String ADAPTHEAD      = name+".adaptHead"; 
-    
-    public final String ADAPTTREEKIND  = name+".adaptTreeKind";
-    public final String ADAPTTRANSKIND = name+".adaptTransKind";
+	/** settings for speaker independent training */
+	public final String CONFIGUREFILE = name + ".configureFile";
+	public final String SPEAKER = name + ".speaker";
+	public final String DATASET = name + ".dataSet";
+	public final String LOWERF0 = name + ".lowerF0";
+	public final String UPPERF0 = name + ".upperF0";
+	public final String NUMTESTFILES = name + ".numTestFiles";
 
-     
-    
-    public String getName(){
-        return name;
-    }
-    
-    /**
-     * Get the map of properties2values
-     * containing the default values
-     * @return map of props2values
-     */
-    public SortedMap<String,String> getDefaultProps(DatabaseLayout db){
-        this.db = db;
-       if (props == null){
-           props = new TreeMap<String,String>();
-           String rootdir = db.getProp(db.ROOTDIR);
-           
-           props.put(CONFIGUREFILE, rootdir+"hts/configure");           
-           props.put(SPEAKER,       "slt");
-           props.put(DATASET,       "cmu_us_arctic");
-           props.put(LOWERF0,       "110");
-           props.put(UPPERF0,       "280");
-           props.put(NUMTESTFILES,  "10");          
-           props.put(VER,         "1");
-           props.put(QNUM,        "001");
-           
-           //Frame period in point 80 for 16Khz; 240 for 48Khz (Frame period in point = sampRate*0.005sec)");
-           int sampRate = db.getSamplingRate();
-           int frameLen   = (int) Math.round(sampRate*0.025);
-           int frameShift = (int) Math.round(sampRate*0.005);           
-           props.put(SAMPFREQ,    db.getProp(DatabaseLayout.SAMPLINGRATE));
-           props.put(FRAMELEN,    Integer.toString(frameLen)); 
-           props.put(FRAMESHIFT,  Integer.toString(frameShift));
-           
-           if(sampRate >= 48000){
-             props.put(FFTLEN, "2048");
-             props.put(FREQWARP, "0.55"); // default HTS-2.2
-           } else if(sampRate >= 44100){
-             props.put(FFTLEN, "2048");
-             props.put(FREQWARP, "0.53");
-           } else if(sampRate >= 22050){
-             props.put(FFTLEN, "1024");
-             props.put(FREQWARP, "0.45");
-           } else if(sampRate >= 16000){
-             props.put(FFTLEN, "512");
-             props.put(FREQWARP, "0.42");  // default HTS-2.1
-           } else if(sampRate >= 12000){
-             props.put(FFTLEN, "512");
-             props.put(FREQWARP, "0.37");
-           } else if(sampRate >= 10000){
-             props.put(FFTLEN, "512");
-             props.put(FREQWARP, "0.35");
-           } else {// sampRate >= 8000)  
-        	 props.put(FFTLEN, "256");
-             props.put(FREQWARP, "0.31");  
-           }
-                      
-           props.put(WINDOWTYPE,  "1");
-           props.put(NORMALIZE,   "1");                             
-           props.put(GAMMA,       "0");           
-           props.put(MGCORDER,    "34");
-           props.put(STRORDER,    "5");
-           
-           if(sampRate >= 48000){
-             props.put(STRFILTERNAME,    "filters/mix_excitation_5filters_199taps_48Kz.txt");
-           } else {
-        	 props.put(STRFILTERNAME,    "filters/mix_excitation_5filters_99taps_16Kz.txt");  
-           }
-           props.put(MGCBANDWIDTH,    "35");
-           props.put(STRBANDWIDTH,    "5");
-           props.put(LF0BANDWIDTH,    "1");
-          
-           props.put(LNGAIN,      "1");
-           props.put(NSTATE,      "5");
-           props.put(NITER,       "5"); 
-             
-           /** settings for HTS ADAPT training scripts */
-           props.put(ADAPTTRAINSPKR,     "'bdl clb jmk rms'");
-           props.put(ADAPTSPKR,     "slt");           
-           //props.put(F0_RANGES,     "'awb 40 280  bdl 40 280  clb 80 350  jmk 40 280  rms 40 280  slt 80 350'");
-           props.put(ADAPTF0_RANGES,     "'bdl 40 210 clb 130 260 jmk 50 180 rms 40 200 slt 110 280'");
-           props.put(ADAPTSPKRMASK,      "*/cmu_us_arctic_%%%_*");
-           props.put(ADAPTHEAD,     "b05");
-           props.put(NUMTESTFILES,  "5");
-               
-           props.put(ADAPTTREEKIND,        "dec");
-           props.put(ADAPTTRANSKIND,       "feat");  
-           
-           props.put(ADAPTSCRIPTS, "false");
-       }
-       
-       return props;
-       }
-    
-    protected void setupHelp(){
-        props2Help = new TreeMap<String,String>();
-        
-        props2Help.put(CONFIGUREFILE, "Path and name of configure file.");
-        props2Help.put(SPEAKER,       "speaker name (default=slt)");
-        props2Help.put(DATASET,       "dataset (default=cmu_us_arctic)");
-        props2Help.put(LOWERF0,       "Lower limit for F0 extraction in Hz (default slt=80 female=80, male=40)");
-        props2Help.put(UPPERF0,       "Upper limit for F0 extraction in Hz (default slt=350 female=350, male=280)");
-        props2Help.put(NUMTESTFILES,  "Number of test files used for testing, these are copied from phonefeatures set.");
-            
-        props2Help.put(VER,         "version number of this setting (default=1)");
-        props2Help.put(QNUM,        "question set number (default='001')");
-        props2Help.put(SAMPFREQ,      "Sampling frequency in Hz (default=48000)");
-        props2Help.put(FRAMELEN,      "Frame length in point (16Khz: 400, 48Khz:1200 default=1200)");
-        props2Help.put(FRAMESHIFT,    "Frame shift in point (16Khz: 80, 48Khz: 240, default=240)");
-        props2Help.put(WINDOWTYPE,  "Window type -> 0: Blackman 1: Hamming 2: Hanning (default=1)");
-        props2Help.put(NORMALIZE,   "Normalization -> 0: none 1: by power 2: by magnitude (default=1)");
-        props2Help.put(FFTLEN,      "FFT length in point (16KHz: 512, 48KHz: 2048,  default=2048)");
-        props2Help.put(FREQWARP,      "Frequency warping factor +" +
-                "8000  FREQWARP=0.31 " +
-                "10000 FREQWARP=0.35 " +
-                "12000 FREQWARP=0.37 " +
-                "16000 FREQWARP=0.42 " +
-                "22050 FREQWARP=0.45 " +
-                "32000 FREQWARP=0.45 " +
-                "44100 FREQWARP=0.53 " +
-                "48000 FREQWARP=0.55  default=0.55)");
-        props2Help.put(GAMMA,       "Pole/Zero weight factor (0: mel-cepstral analysis 1: LPC analysis 2,3,...,N: mel-generalized cepstral (MGC) analysis) (default=0)");
-        props2Help.put(MGCORDER,    "Order of MGC analysis (default=24 for cepstral form, default=12 for LSP form)");
-        props2Help.put(STRORDER,    "Order of strengths analysis (default=5 for 5 filter bands)");
-        props2Help.put(STRFILTERNAME, "Name of file containig the filters for voicing strengths analysis in mixed excitation " +
-        		                      "(default 48Kz = filters/mix_excitation_5filters_199taps_48Kz.txt " +
-        		                      " default 16Kz = filters/mix_excitation_5filters_99taps_16Kz.txt). More than 5 filters can be defined so STRORDER should" +
-        		                      "be defined accordingly."); 
-        props2Help.put(MGCBANDWIDTH,    "band width for MGC transforms (default=24 for cepstral form, derault=1 for LSP form)");
-        props2Help.put(STRBANDWIDTH,    "band width for STR transforms (default=5)");
-        props2Help.put(LF0BANDWIDTH,    "band width for log F0 transforms (default=1)");
-        
-        props2Help.put(LNGAIN,      "Use logarithmic gain instead of linear gain (default=1)");
-        props2Help.put(NSTATE,      "number of HMM states (default=5)");
-        props2Help.put(NITER,       "number of iterations of embedded training (default=5)");        
-        
-        // for scripts adapt
-        props2Help.put(ADAPTTRAINSPKR,"only ADAPTSCRIPTS: speakers for training (default='awb bdl clb jmk rms')");
-        props2Help.put(ADAPTSPKR,     "only ADAPTSCRIPTS: speakers for adaptation (default=slt)");
-        props2Help.put(ADAPTSPKRMASK, "only ADAPTSCRIPTS: speaker name pattern (mask for file names, -h option in HERest) (default=*/cmu_us_arctic_%%%_*)");
-        props2Help.put(ADAPTHEAD,     "only ADAPTSCRIPTS: file name header for adaptation data (default=b05)");      
-        props2Help.put(ADAPTF0_RANGES,"only ADAPTSCRIPTS: F0 search ranges (spkr1 lower1 upper1  spkr2 lower2 upper2...). " +
-                                      "only ADAPTSCRIPTS: Order of speakers in F0_RANGES should be equal to that in ALLSPKR=$(TRAINSPKR) $(ADAPTSPKR)" +
-                                      "(default='bdl 40 210 clb 130 260 jmk 50 180 rms 40 200 slt 110 280')");        
-        props2Help.put(ADAPTTREEKIND, "only ADAPTSCRIPTS: regression class tree kind (dec: decision tree, reg: regression tree, default=dec)");
-        props2Help.put(ADAPTTRANSKIND,"only ADAPTSCRIPTS: adaptation transform kind (mean: MLLRMEAN, cov: MLLRCOV, feat: CMLLR, default=feat)");
-          
-        props2Help.put(ADAPTSCRIPTS, "ADAPTSCRIPTS=false: speaker dependent scripts, ADAPTSCRIPTS=true: speaker adaptation/adaptive scripts.  ");     
-    }
+	public final String VER = name + ".version";
+	public final String QNUM = name + ".questionsNum";
+	public final String FRAMELEN = name + ".frameLen";
+	public final String FRAMESHIFT = name + ".frameShift";
+	public final String WINDOWTYPE = name + ".windowType";
+	public final String NORMALIZE = name + ".normalize";
+	public final String FFTLEN = name + ".fftLen";
+	public final String FREQWARP = name + ".freqWarp";
+	public final String GAMMA = name + ".gamma";
+	public final String MGCORDER = name + ".mgcOrder";
+	public final String STRORDER = name + ".strOrder";
+	public final String STRFILTERNAME = name + ".strFilterFileName";
+	public final String MGCBANDWIDTH = name + ".mgcBandWidth";
+	public final String STRBANDWIDTH = name + ".strBandWidth";
+	public final String LF0BANDWIDTH = name + ".lf0BandWidth";
 
-    
-    
-    /**
-     * Do the computations required by this component.
-     * 
-     * @return true on success, false on failure
-     */
-    public boolean compute() throws Exception{
-        
-        System.out.println("\nChecking directories and files for running HTS training scripts...");
-        
-        String filedir = db.getProp(DatabaseLayout.ROOTDIR);
-        String cmdLine;
-        boolean speech_transcriptions = true;
- 
-       
-       File dirWav  = new File(filedir + "wav");
-       File dirText = new File(filedir + "text");
-       File dirRaw  = new File(filedir + "hts/data/raw");
-       File dirUtt  = new File(filedir + "hts/data/utts");
-       File dirFea  = new File(filedir + "phonefeatures");
-       File dirLab  = new File(filedir + "phonelab");
+	public final String LNGAIN = name + ".lnGain";
+	public final String SAMPFREQ = name + ".sampfreq";
+	public final String NSTATE = name + ".numState";
+	public final String NITER = name + ".numIterations";
+
+	/** settings for HTS ADAPT training scripts */
+	public final String ADAPTTRAINSPKR = name + ".adaptTrainSpkr";
+	public final String ADAPTSPKR = name + ".adaptSpkr";
+	public final String ADAPTF0_RANGES = name + ".adaptF0Ranges";
+	public final String ADAPTSPKRMASK = name + ".adaptSpkrMask";
+	public final String ADAPTHEAD = name + ".adaptHead";
+
+	public final String ADAPTTREEKIND = name + ".adaptTreeKind";
+	public final String ADAPTTRANSKIND = name + ".adaptTransKind";
+
+	public String getName() {
+		return name;
+	}
+
+	/**
+	 * Get the map of properties2values containing the default values
+	 * 
+	 * @return map of props2values
+	 */
+	public SortedMap<String, String> getDefaultProps(DatabaseLayout db) {
+		this.db = db;
+		if (props == null) {
+			props = new TreeMap<String, String>();
+			String rootdir = db.getProp(db.ROOTDIR);
+
+			props.put(CONFIGUREFILE, rootdir + "hts/configure");
+			props.put(SPEAKER, "slt");
+			props.put(DATASET, "cmu_us_arctic");
+			props.put(LOWERF0, "110");
+			props.put(UPPERF0, "280");
+			props.put(NUMTESTFILES, "10");
+			props.put(VER, "1");
+			props.put(QNUM, "001");
+
+			// Frame period in point 80 for 16Khz; 240 for 48Khz (Frame period in point = sampRate*0.005sec)");
+			int sampRate = db.getSamplingRate();
+			int frameLen = (int) Math.round(sampRate * 0.025);
+			int frameShift = (int) Math.round(sampRate * 0.005);
+			props.put(SAMPFREQ, db.getProp(DatabaseLayout.SAMPLINGRATE));
+			props.put(FRAMELEN, Integer.toString(frameLen));
+			props.put(FRAMESHIFT, Integer.toString(frameShift));
+
+			if (sampRate >= 48000) {
+				props.put(FFTLEN, "2048");
+				props.put(FREQWARP, "0.55"); // default HTS-2.2
+			} else if (sampRate >= 44100) {
+				props.put(FFTLEN, "2048");
+				props.put(FREQWARP, "0.53");
+			} else if (sampRate >= 22050) {
+				props.put(FFTLEN, "1024");
+				props.put(FREQWARP, "0.45");
+			} else if (sampRate >= 16000) {
+				props.put(FFTLEN, "512");
+				props.put(FREQWARP, "0.42"); // default HTS-2.1
+			} else if (sampRate >= 12000) {
+				props.put(FFTLEN, "512");
+				props.put(FREQWARP, "0.37");
+			} else if (sampRate >= 10000) {
+				props.put(FFTLEN, "512");
+				props.put(FREQWARP, "0.35");
+			} else {// sampRate >= 8000)
+				props.put(FFTLEN, "256");
+				props.put(FREQWARP, "0.31");
+			}
+
+			props.put(WINDOWTYPE, "1");
+			props.put(NORMALIZE, "1");
+			props.put(GAMMA, "0");
+			props.put(MGCORDER, "34");
+			props.put(STRORDER, "5");
+
+			if (sampRate >= 48000) {
+				props.put(STRFILTERNAME, "filters/mix_excitation_5filters_199taps_48Kz.txt");
+			} else {
+				props.put(STRFILTERNAME, "filters/mix_excitation_5filters_99taps_16Kz.txt");
+			}
+			props.put(MGCBANDWIDTH, "35");
+			props.put(STRBANDWIDTH, "5");
+			props.put(LF0BANDWIDTH, "1");
+
+			props.put(LNGAIN, "1");
+			props.put(NSTATE, "5");
+			props.put(NITER, "5");
+
+			/** settings for HTS ADAPT training scripts */
+			props.put(ADAPTTRAINSPKR, "'bdl clb jmk rms'");
+			props.put(ADAPTSPKR, "slt");
+			// props.put(F0_RANGES, "'awb 40 280  bdl 40 280  clb 80 350  jmk 40 280  rms 40 280  slt 80 350'");
+			props.put(ADAPTF0_RANGES, "'bdl 40 210 clb 130 260 jmk 50 180 rms 40 200 slt 110 280'");
+			props.put(ADAPTSPKRMASK, "*/cmu_us_arctic_%%%_*");
+			props.put(ADAPTHEAD, "b05");
+			props.put(NUMTESTFILES, "5");
+
+			props.put(ADAPTTREEKIND, "dec");
+			props.put(ADAPTTRANSKIND, "feat");
+
+			props.put(ADAPTSCRIPTS, "false");
+		}
+
+		return props;
+	}
+
+	protected void setupHelp() {
+		props2Help = new TreeMap<String, String>();
+
+		props2Help.put(CONFIGUREFILE, "Path and name of configure file.");
+		props2Help.put(SPEAKER, "speaker name (default=slt)");
+		props2Help.put(DATASET, "dataset (default=cmu_us_arctic)");
+		props2Help.put(LOWERF0, "Lower limit for F0 extraction in Hz (default slt=80 female=80, male=40)");
+		props2Help.put(UPPERF0, "Upper limit for F0 extraction in Hz (default slt=350 female=350, male=280)");
+		props2Help.put(NUMTESTFILES, "Number of test files used for testing, these are copied from phonefeatures set.");
+
+		props2Help.put(VER, "version number of this setting (default=1)");
+		props2Help.put(QNUM, "question set number (default='001')");
+		props2Help.put(SAMPFREQ, "Sampling frequency in Hz (default=48000)");
+		props2Help.put(FRAMELEN, "Frame length in point (16Khz: 400, 48Khz:1200 default=1200)");
+		props2Help.put(FRAMESHIFT, "Frame shift in point (16Khz: 80, 48Khz: 240, default=240)");
+		props2Help.put(WINDOWTYPE, "Window type -> 0: Blackman 1: Hamming 2: Hanning (default=1)");
+		props2Help.put(NORMALIZE, "Normalization -> 0: none 1: by power 2: by magnitude (default=1)");
+		props2Help.put(FFTLEN, "FFT length in point (16KHz: 512, 48KHz: 2048,  default=2048)");
+		props2Help.put(FREQWARP, "Frequency warping factor +" + "8000  FREQWARP=0.31 " + "10000 FREQWARP=0.35 "
+				+ "12000 FREQWARP=0.37 " + "16000 FREQWARP=0.42 " + "22050 FREQWARP=0.45 " + "32000 FREQWARP=0.45 "
+				+ "44100 FREQWARP=0.53 " + "48000 FREQWARP=0.55  default=0.55)");
+		props2Help
+				.put(GAMMA,
+						"Pole/Zero weight factor (0: mel-cepstral analysis 1: LPC analysis 2,3,...,N: mel-generalized cepstral (MGC) analysis) (default=0)");
+		props2Help.put(MGCORDER, "Order of MGC analysis (default=24 for cepstral form, default=12 for LSP form)");
+		props2Help.put(STRORDER, "Order of strengths analysis (default=5 for 5 filter bands)");
+		props2Help
+				.put(STRFILTERNAME,
+						"Name of file containig the filters for voicing strengths analysis in mixed excitation "
+								+ "(default 48Kz = filters/mix_excitation_5filters_199taps_48Kz.txt "
+								+ " default 16Kz = filters/mix_excitation_5filters_99taps_16Kz.txt). More than 5 filters can be defined so STRORDER should"
+								+ "be defined accordingly.");
+		props2Help.put(MGCBANDWIDTH, "band width for MGC transforms (default=24 for cepstral form, derault=1 for LSP form)");
+		props2Help.put(STRBANDWIDTH, "band width for STR transforms (default=5)");
+		props2Help.put(LF0BANDWIDTH, "band width for log F0 transforms (default=1)");
+
+		props2Help.put(LNGAIN, "Use logarithmic gain instead of linear gain (default=1)");
+		props2Help.put(NSTATE, "number of HMM states (default=5)");
+		props2Help.put(NITER, "number of iterations of embedded training (default=5)");
+
+		// for scripts adapt
+		props2Help.put(ADAPTTRAINSPKR, "only ADAPTSCRIPTS: speakers for training (default='awb bdl clb jmk rms')");
+		props2Help.put(ADAPTSPKR, "only ADAPTSCRIPTS: speakers for adaptation (default=slt)");
+		props2Help
+				.put(ADAPTSPKRMASK,
+						"only ADAPTSCRIPTS: speaker name pattern (mask for file names, -h option in HERest) (default=*/cmu_us_arctic_%%%_*)");
+		props2Help.put(ADAPTHEAD, "only ADAPTSCRIPTS: file name header for adaptation data (default=b05)");
+		props2Help
+				.put(ADAPTF0_RANGES,
+						"only ADAPTSCRIPTS: F0 search ranges (spkr1 lower1 upper1  spkr2 lower2 upper2...). "
+								+ "only ADAPTSCRIPTS: Order of speakers in F0_RANGES should be equal to that in ALLSPKR=$(TRAINSPKR) $(ADAPTSPKR)"
+								+ "(default='bdl 40 210 clb 130 260 jmk 50 180 rms 40 200 slt 110 280')");
+		props2Help.put(ADAPTTREEKIND,
+				"only ADAPTSCRIPTS: regression class tree kind (dec: decision tree, reg: regression tree, default=dec)");
+		props2Help.put(ADAPTTRANSKIND,
+				"only ADAPTSCRIPTS: adaptation transform kind (mean: MLLRMEAN, cov: MLLRCOV, feat: CMLLR, default=feat)");
+
+		props2Help.put(ADAPTSCRIPTS,
+				"ADAPTSCRIPTS=false: speaker dependent scripts, ADAPTSCRIPTS=true: speaker adaptation/adaptive scripts.  ");
+	}
+
+	/**
+	 * Do the computations required by this component.
+	 * 
+	 * @return true on success, false on failure
+	 */
+	public boolean compute() throws Exception {
+
+		System.out.println("\nChecking directories and files for running HTS training scripts...");
+
+		String filedir = db.getProp(DatabaseLayout.ROOTDIR);
+		String cmdLine;
+		boolean speech_transcriptions = true;
+
+		File dirWav = new File(filedir + "wav");
+		File dirText = new File(filedir + "text");
+		File dirRaw = new File(filedir + "hts/data/raw");
+		File dirUtt = new File(filedir + "hts/data/utts");
+		File dirFea = new File(filedir + "phonefeatures");
+		File dirLab = new File(filedir + "phonelab");
+
+		if (getProp(ADAPTSCRIPTS).contentEquals("false")) {
+			// Check if wav directory exist and have files
+			if (!dirWav.exists() || dirWav.list().length == 0 || !dirRaw.exists() || dirRaw.list().length == 0) {
+				System.out.println("Problem with wav and hts/data/raw directories: wav files and raw files do not exist"
+						+ " in current directory: " + filedir);
+				speech_transcriptions = false;
+			}
+			// check if hts/data/raw directory exist and have files
+			if (!dirWav.exists() || dirWav.list().length == 0 || !dirRaw.exists() || dirRaw.list().length == 0) {
+				System.out.println("Problem with wav and hts/data/raw directories: wav files and raw files do not exist"
+						+ " in current directory: " + filedir);
+				speech_transcriptions = false;
+			}
+			// Check if text directory exist and have files
+			if ((!dirText.exists() || dirText.list().length == 0) && (!dirUtt.exists() || dirUtt.list().length == 0)) {
+				System.out
+						.println("Problem with transcription directories text or hts/data/utts (Festival format): utts files and text files do not exist"
+								+ " in current directory: " + filedir);
+				System.out
+						.println(" the transcriptions in the directory text will be used to generate the phonelab directory, if there are no hts/data/utts files"
+								+ "(in Festival format), please provide the transcriptions of the files you are going to use for trainning.");
+				speech_transcriptions = false;
+			}
+			// Check if phonefeatures and phonelab directory exist and have files
+			if ((!dirFea.exists() || dirFea.list().length == 0) && (!dirLab.exists() || dirLab.list().length == 0)) {
+				System.out.println("Problems with directories phonefeatures or phonelab, they do not exist or they are empty.");
+				speech_transcriptions = false;
+			}
+		} else { // configuration for ADAPTSCRIPT
+			// Get the speakers directories
+			File dirSpeakersFea = new File(filedir + "/phonefeatures");
+			File dirSpeakersLab = new File(filedir + "/phonelab");
+
+			String[] speakers;
+			if (dirSpeakersFea.exists() && dirSpeakersFea.list().length > 0 && dirSpeakersLab.exists()
+					&& dirSpeakersLab.list().length > 0) {
+				speakers = dirSpeakersFea.list();
+				for (int i = 0; i < speakers.length; i++) {
+					File dirSpeakerFea = new File(filedir + "/phonefeatures/" + speakers[i]);
+					File dirSpeakerLab = new File(filedir + "/phonelab/" + speakers[i]);
+					if (dirSpeakerFea.exists() && dirSpeakerFea.list().length > 0 && dirSpeakerLab.exists()
+							&& dirSpeakerLab.list().length > 0) {
+						speech_transcriptions = true;
+					} else {
+						System.out.println("Error: directories " + filedir + "/phonefeatures/" + speakers[i] + " and/or "
+								+ filedir + "/phonelab/" + speakers[i] + " do not contain files.");
+						speech_transcriptions = false;
+						break;
+					}
+				}
+			} else {
+				System.out.println("Error: directories " + filedir + "phonefeatures and/or " + filedir
+						+ "phonelab do not contain files.");
+				speech_transcriptions = false;
+			}
+		}
+
+		if (speech_transcriptions) {
+
+			if (getProp(ADAPTSCRIPTS).contentEquals("false")) {
+				/* if previous files and directories exist then run configure */
+				/* first it should go to the hts directory and there run ./configure */
+				System.out.println("Running make configure: ");
+				cmdLine = "chmod +x " + getProp(CONFIGUREFILE);
+				General.launchProc(cmdLine, "configure", filedir);
+
+				cmdLine = "cd " + filedir + "hts\n" + getProp(CONFIGUREFILE) + " --with-tcl-search-path="
+						+ db.getExternal(DatabaseLayout.TCLPATH) + " --with-sptk-search-path="
+						+ db.getExternal(DatabaseLayout.SPTKPATH) + " --with-hts-search-path="
+						+ db.getExternal(DatabaseLayout.HTSPATH) + " --with-hts-engine-search-path="
+						+ db.getExternal(DatabaseLayout.HTSENGINEPATH) + " --with-sox-search-path="
+						+ db.getExternal(DatabaseLayout.SOXPATH) + " SPEAKER=" + getProp(SPEAKER) + " DATASET="
+						+ getProp(DATASET) + " LOWERF0=" + getProp(LOWERF0) + " UPPERF0=" + getProp(UPPERF0) + " VER="
+						+ getProp(VER) + " QNUM=" + getProp(QNUM) + " FRAMELEN=" + getProp(FRAMELEN) + " FRAMESHIFT="
+						+ getProp(FRAMESHIFT) + " WINDOWTYPE=" + getProp(WINDOWTYPE) + " NORMALIZE=" + getProp(NORMALIZE)
+						+ " FFTLEN=" + getProp(FFTLEN) + " FREQWARP=" + getProp(FREQWARP) + " GAMMA=" + getProp(GAMMA)
+						+ " MGCORDER=" + getProp(MGCORDER) + " STRORDER=" + getProp(STRORDER) + " STRFILTERNAME="
+						+ getProp(STRFILTERNAME) + " LNGAIN=" + getProp(LNGAIN) + " SAMPFREQ=" + getProp(SAMPFREQ) + " NSTATE="
+						+ getProp(NSTATE) + " NITER=" + getProp(NITER);
+
+			} else {
+				/* if previous files and directories exist then run configure */
+				System.out.println("Running make configure: ");
+				cmdLine = "chmod +x " + getProp(CONFIGUREFILE);
+				General.launchProc(cmdLine, "configure", filedir);
+
+				cmdLine = "cd " + filedir + "hts\n" + getProp(CONFIGUREFILE) + " --with-tcl-search-path="
+						+ db.getExternal(DatabaseLayout.TCLPATH) + " --with-sptk-search-path="
+						+ db.getExternal(DatabaseLayout.SPTKPATH) + " --with-hts-search-path="
+						+ db.getExternal(DatabaseLayout.HTSPATH) + " --with-hts-engine-search-path="
+						+ db.getExternal(DatabaseLayout.HTSENGINEPATH) + " --with-sox-search-path="
+						+ db.getExternal(DatabaseLayout.SOXPATH) + " SPEAKER=" + getProp(SPEAKER) + " DATASET="
+						+ getProp(DATASET) + " TRAINSPKR=" + getProp(ADAPTTRAINSPKR) + " ADAPTSPKR=" + getProp(ADAPTSPKR)
+						+ " F0_RANGES=" + getProp(ADAPTF0_RANGES) + " SPKRMASK=" + getProp(ADAPTSPKRMASK) + " ADAPTHEAD="
+						+ getProp(ADAPTHEAD) + " VER=" + getProp(VER) + " QNUM=" + getProp(QNUM) + " FRAMELEN="
+						+ getProp(FRAMELEN) + " FRAMESHIFT=" + getProp(FRAMESHIFT) + " WINDOWTYPE=" + getProp(WINDOWTYPE)
+						+ " NORMALIZE=" + getProp(NORMALIZE) + " FFTLEN=" + getProp(FFTLEN) + " FREQWARP=" + getProp(FREQWARP)
+						+ " GAMMA=" + getProp(GAMMA) + " MGCORDER=" + getProp(MGCORDER) + " STRORDER=" + getProp(STRORDER)
+						+ " STRFILTERNAME=" + getProp(STRFILTERNAME) + " LNGAIN=" + getProp(LNGAIN) + " SAMPFREQ="
+						+ getProp(SAMPFREQ) + " NSTATE=" + getProp(NSTATE) + " NITER=" + getProp(NITER) + " MGCBANDWIDTH="
+						+ getProp(MGCBANDWIDTH) + " STRBANDWIDTH=" + getProp(STRBANDWIDTH) + " LF0BANDWIDTH="
+						+ getProp(LF0BANDWIDTH) + " TREEKIND=" + getProp(ADAPTTREEKIND) + " TRANSKIND=" + getProp(ADAPTTRANSKIND);
+			}
+
+			General.launchBatchProc(cmdLine, "Configure", filedir);
+
+		} else {
+			System.out.println("Problems with directories phonefeatures or phonelab, they do not exist or they are empty.");
+			System.out.println("Problems with directories wav, text or hts/data/raw, they do not exist or they are empty.");
+			return false;
+		}
+
+		return true;
+
+	}
+
+	/**
+	 * Provide the progress of computation, in percent, or -1 if that feature is not implemented.
+	 * 
+	 * @return -1 if not implemented, or an integer between 0 and 100.
+	 */
+	public int getProgress() {
+		return -1;
+	}
 
-       if(getProp(ADAPTSCRIPTS).contentEquals("false")) { 
-         // Check if wav directory exist and have files
-         if( !dirWav.exists() || dirWav.list().length == 0 || !dirRaw.exists() || dirRaw.list().length == 0 ){ 
-           System.out.println("Problem with wav and hts/data/raw directories: wav files and raw files do not exist" +
-                " in current directory: " + filedir);
-           speech_transcriptions = false;
-         }        
-         // check if hts/data/raw directory exist and have files
-         if( !dirWav.exists() || dirWav.list().length == 0 || !dirRaw.exists() || dirRaw.list().length == 0 ){
-           System.out.println("Problem with wav and hts/data/raw directories: wav files and raw files do not exist" +
-                " in current directory: " + filedir);
-           speech_transcriptions = false;
-         }        
-         // Check if text directory exist and have files
-         if( ( !dirText.exists() || dirText.list().length == 0 ) && ( !dirUtt.exists() || dirUtt.list().length == 0 ) ){
-           System.out.println("Problem with transcription directories text or hts/data/utts (Festival format): utts files and text files do not exist" +
-                " in current directory: " + filedir);
-           System.out.println(" the transcriptions in the directory text will be used to generate the phonelab directory, if there are no hts/data/utts files" +
-                   "(in Festival format), please provide the transcriptions of the files you are going to use for trainning.");
-           speech_transcriptions = false;
-         } 
-         // Check if phonefeatures and phonelab directory exist and have files
-         if( ( !dirFea.exists() || dirFea.list().length == 0 ) && ( !dirLab.exists() || dirLab.list().length == 0 ) ){
-           System.out.println("Problems with directories phonefeatures or phonelab, they do not exist or they are empty.");
-           speech_transcriptions = false;
-         }
-       } else {  // configuration for ADAPTSCRIPT
-           // Get the speakers directories
-           File dirSpeakersFea = new File(filedir + "/phonefeatures");
-           File dirSpeakersLab = new File(filedir + "/phonelab");
-           
-           String[] speakers;
-           if(dirSpeakersFea.exists() && dirSpeakersFea.list().length > 0 && dirSpeakersLab.exists() && dirSpeakersLab.list().length > 0 ){ 
-               speakers = dirSpeakersFea.list();
-               for(int i=0; i<speakers.length; i++){
-                 File dirSpeakerFea = new File(filedir + "/phonefeatures/" + speakers[i]);
-                 File dirSpeakerLab = new File(filedir + "/phonelab/" + speakers[i]);
-                 if(dirSpeakerFea.exists() && dirSpeakerFea.list().length > 0 && dirSpeakerLab.exists() && dirSpeakerLab.list().length > 0 ){
-                   speech_transcriptions = true;     
-                 } else {
-                   System.out.println("Error: directories " + filedir + "/phonefeatures/" + speakers[i] + 
-                           " and/or " + filedir + "/phonelab/" + speakers[i] + " do not contain files." );
-                   speech_transcriptions = false;
-                   break;
-                 }
-               }
-           } else {            
-               System.out.println("Error: directories " + filedir + "phonefeatures and/or " + filedir + "phonelab do not contain files." );
-               speech_transcriptions = false;
-           }               
-       }
-       
-       if(speech_transcriptions){
-           
-       if(getProp(ADAPTSCRIPTS).contentEquals("false")) {                    
-         /* if previous files and directories exist then run configure */
-         /* first it should go to the hts directory and there run ./configure*/
-         System.out.println("Running make configure: ");
-         cmdLine = "chmod +x " + getProp(CONFIGUREFILE);
-         General.launchProc(cmdLine, "configure", filedir);
-       
-         cmdLine = "cd " + filedir + "hts\n" + 
-         getProp(CONFIGUREFILE) +
-         " --with-tcl-search-path="        + db.getExternal(DatabaseLayout.TCLPATH) +
-         " --with-sptk-search-path="       + db.getExternal(DatabaseLayout.SPTKPATH) +
-         " --with-hts-search-path="        + db.getExternal(DatabaseLayout.HTSPATH) +
-         " --with-hts-engine-search-path=" + db.getExternal(DatabaseLayout.HTSENGINEPATH) +
-         " --with-sox-search-path="        + db.getExternal(DatabaseLayout.SOXPATH) +
-         " SPEAKER=" + getProp(SPEAKER) +
-         " DATASET=" + getProp(DATASET) +
-         " LOWERF0=" + getProp(LOWERF0) +
-         " UPPERF0=" + getProp(UPPERF0) +
-         " VER=" + getProp(VER) +
-         " QNUM=" + getProp(QNUM) +
-         " FRAMELEN=" + getProp(FRAMELEN) +
-         " FRAMESHIFT=" + getProp(FRAMESHIFT) +
-         " WINDOWTYPE=" + getProp(WINDOWTYPE) +
-         " NORMALIZE=" + getProp(NORMALIZE) +
-         " FFTLEN=" + getProp(FFTLEN) +
-         " FREQWARP=" + getProp(FREQWARP) +
-         " GAMMA=" + getProp(GAMMA) +
-         " MGCORDER=" + getProp(MGCORDER) +
-         " STRORDER=" + getProp(STRORDER) +
-         " STRFILTERNAME=" + getProp(STRFILTERNAME) +
-         " LNGAIN=" + getProp(LNGAIN) +
-         " SAMPFREQ=" + getProp(SAMPFREQ) +
-         " NSTATE=" + getProp(NSTATE) +
-         " NITER=" + getProp(NITER);
-         
-       } else {            
-             /* if previous files and directories exist then run configure */
-           System.out.println("Running make configure: ");
-           cmdLine = "chmod +x " + getProp(CONFIGUREFILE);
-           General.launchProc(cmdLine, "configure", filedir);
-         
-           cmdLine = "cd " + filedir + "hts\n" +
-             getProp(CONFIGUREFILE) +
-             " --with-tcl-search-path="        + db.getExternal(DatabaseLayout.TCLPATH) +
-             " --with-sptk-search-path="       + db.getExternal(DatabaseLayout.SPTKPATH) +
-             " --with-hts-search-path="        + db.getExternal(DatabaseLayout.HTSPATH) +
-             " --with-hts-engine-search-path=" + db.getExternal(DatabaseLayout.HTSENGINEPATH) +
-             " --with-sox-search-path="        + db.getExternal(DatabaseLayout.SOXPATH) +
-             " SPEAKER=" + getProp(SPEAKER) +
-             " DATASET=" + getProp(DATASET) +
-             " TRAINSPKR=" + getProp(ADAPTTRAINSPKR) + 
-             " ADAPTSPKR=" + getProp(ADAPTSPKR) + 
-             " F0_RANGES=" + getProp(ADAPTF0_RANGES) + 
-             " SPKRMASK=" + getProp(ADAPTSPKRMASK) + 
-             " ADAPTHEAD=" + getProp(ADAPTHEAD) +          
-             " VER=" + getProp(VER) +
-             " QNUM=" + getProp(QNUM) +
-             " FRAMELEN=" + getProp(FRAMELEN) +
-             " FRAMESHIFT=" + getProp(FRAMESHIFT) +
-             " WINDOWTYPE=" + getProp(WINDOWTYPE) +
-             " NORMALIZE=" + getProp(NORMALIZE) +
-             " FFTLEN=" + getProp(FFTLEN) +
-             " FREQWARP=" + getProp(FREQWARP) +
-             " GAMMA=" + getProp(GAMMA) +
-             " MGCORDER=" + getProp(MGCORDER) +
-             " STRORDER=" + getProp(STRORDER) +
-             " STRFILTERNAME=" + getProp(STRFILTERNAME) +
-             " LNGAIN=" + getProp(LNGAIN) +
-             " SAMPFREQ=" + getProp(SAMPFREQ) +
-             " NSTATE=" + getProp(NSTATE) +
-             " NITER=" + getProp(NITER) +
-             " MGCBANDWIDTH=" + getProp(MGCBANDWIDTH) +
-             " STRBANDWIDTH=" + getProp(STRBANDWIDTH) +
-             " LF0BANDWIDTH=" + getProp(LF0BANDWIDTH) +
-             " TREEKIND=" + getProp(ADAPTTREEKIND) +
-             " TRANSKIND=" + getProp(ADAPTTRANSKIND);
-         }
-              
-         General.launchBatchProc(cmdLine, "Configure", filedir);
-             
-       } else {
-         System.out.println("Problems with directories phonefeatures or phonelab, they do not exist or they are empty.");
-         System.out.println("Problems with directories wav, text or hts/data/raw, they do not exist or they are empty.");
-         return false;
-       }
-       
-       return true;
-       
-    }
-    
-    
-    /**
-     * Provide the progress of computation, in percent, or -1 if
-     * that feature is not implemented.
-     * @return -1 if not implemented, or an integer between 0 and 100.
-     */
-    public int getProgress(){
-        return -1;
-    }
-    
 }

marytts-builder/src/main/java/marytts/tools/voiceimport/HMMVoiceDataPreparation.java

@@ -70,339 +70,329 @@ import java.util.TreeMap;
 import marytts.util.io.FileUtils;
 import marytts.util.io.General;
 
-
-
 /**
- * This program was modified from previous version to:
- * 1. copy $MARY_BASE/lib/external/hts directory to the voice building directory
- * 2. check again that all the external necessary programs are installed.
- * 3. check as before that wav and text directories exist and make conversions:
- *    voiceDir/wav -> voiceDir/hts/data/raw 
- *    userProvidedDir/utts (festival format) -> voiceDir/text (one file per transcription)
- *    userProvidedDir/raw move to voiceDir/hts/data/raw
+ * This program was modified from previous version to: 1. copy $MARY_BASE/lib/external/hts directory to the voice building
+ * directory 2. check again that all the external necessary programs are installed. 3. check as before that wav and text
+ * directories exist and make conversions: voiceDir/wav -> voiceDir/hts/data/raw userProvidedDir/utts (festival format) ->
+ * voiceDir/text (one file per transcription) userProvidedDir/raw move to voiceDir/hts/data/raw
  *
  * @author marcela
  *
  */
-public class HMMVoiceDataPreparation extends VoiceImportComponent{
-    private DatabaseLayout db;
-    private String name = "HMMVoiceDataPreparation";
-    public final String ADAPTSCRIPTS  = name + ".adaptScripts";
-    public final String USERRAWDIR    = name + ".userRawDirectory";
-    public final String USERUTTDIR    = name + ".userUttDirectory";        
-    private String marybase; 
-    private String voiceDir;
-    private String soxPath;
-    private String sep;
-    private String dataDir;
-    private String scriptsDir;
+public class HMMVoiceDataPreparation extends VoiceImportComponent {
+	private DatabaseLayout db;
+	private String name = "HMMVoiceDataPreparation";
+	public final String ADAPTSCRIPTS = name + ".adaptScripts";
+	public final String USERRAWDIR = name + ".userRawDirectory";
+	public final String USERUTTDIR = name + ".userUttDirectory";
+	private String marybase;
+	private String voiceDir;
+	private String soxPath;
+	private String sep;
+	private String dataDir;
+	private String scriptsDir;
 
+	public String getName() {
+		return name;
+	}
 
-    public String getName(){
-        return name;
-    }
-    
-    /**
-     * Get the map of properties2values
-     * containing the default values
-     * @return map of props2values
-     */
-    public SortedMap<String,String> getDefaultProps(DatabaseLayout db){
-        this.db = db;
-       if (props == null){
-           props = new TreeMap<String,String>();           
-           props.put(USERRAWDIR, "");
-           props.put(USERUTTDIR, "");
-           props.put(ADAPTSCRIPTS, "false");          
-       }
-       return props;
-       }
-    
-    protected void setupHelp(){
-        props2Help = new TreeMap<String,String>();                
-        props2Help.put(USERRAWDIR, "raw files directory, user provided directory (default empty)");
-        props2Help.put(USERUTTDIR, "utterance directory (transcriptions in festival format), user provided directory (default empty)");   
-        props2Help.put(ADAPTSCRIPTS, "ADAPTSCRIPTS=false: speaker dependent scripts, ADAPTSCRIPTS=true: speaker adaptation/adaptive scripts.  "); 
-    }
+	/**
+	 * Get the map of properties2values containing the default values
+	 * 
+	 * @return map of props2values
+	 */
+	public SortedMap<String, String> getDefaultProps(DatabaseLayout db) {
+		this.db = db;
+		if (props == null) {
+			props = new TreeMap<String, String>();
+			props.put(USERRAWDIR, "");
+			props.put(USERUTTDIR, "");
+			props.put(ADAPTSCRIPTS, "false");
+		}
+		return props;
+	}
 
-       
-    /**
-     * Do the computations required by this component.
-     * 
-     * @return true on success, false on failure
-     */
-    public boolean compute() throws Exception{
-       boolean raw = false;
-       boolean text = false;
-       boolean wav = false;
-       marybase   = db.getProp(DatabaseLayout.MARYBASE); 
-       voiceDir   = db.getProp(DatabaseLayout.ROOTDIR);
-       soxPath    = db.getExternal(DatabaseLayout.SOXPATH);
-       sep        = System.getProperty("file.separator");       
-       dataDir    = voiceDir + "hts" + sep + "data" + sep;
-       scriptsDir = dataDir + "scripts" + sep;
-       
-       // For both speaker indep. or adapt scripts the programs are the same
-       // check again that all the external necessary programs are installed.
-       System.out.println("\nHMMVoiceDataPreparation:\nChecking paths of external programs");
-       if( !checkExternalPaths() )
-           return false;
+	protected void setupHelp() {
+		props2Help = new TreeMap<String, String>();
+		props2Help.put(USERRAWDIR, "raw files directory, user provided directory (default empty)");
+		props2Help.put(USERUTTDIR,
+				"utterance directory (transcriptions in festival format), user provided directory (default empty)");
+		props2Help.put(ADAPTSCRIPTS,
+				"ADAPTSCRIPTS=false: speaker dependent scripts, ADAPTSCRIPTS=true: speaker adaptation/adaptive scripts.  ");
+	}
 
- 
-       if(getProp(ADAPTSCRIPTS).contentEquals("false")) { 
-                 
-       // 1. copy from $MARY_TTS/lib/external/hts directory in the voice building directory
-       String sourceFolder = marybase + sep + "lib" + sep + "external" + sep + "hts";
-       String htsFolder = voiceDir + sep + "hts";
-       FileUtils.copyFolderRecursive(sourceFolder, htsFolder, false);
-              
-       // 2. check as before that wav, raw and text directories exist and are in the correct place     
-       System.out.println("\nChecking wav/raw and text directories and files for running HTS speaker independent training scripts...");
-                            
-       // default locations of directories:
-       String wavDirName  = voiceDir + "wav";           
-       String textDirName = voiceDir + "text";
-       String rawDirName  = dataDir + "raw";
-       String uttsDirName = dataDir + "utts";
-       
-       // 2.1 check raw and wav files:
-       String userRawDirName = getProp(USERRAWDIR);  
-       if( existWithFiles(rawDirName) ) {
-         raw = true;
-         // the raw files should be the same as in wav file, if wav file empty convert from raw --> wav
-         if( !existWithFiles(wavDirName))
-           convertRaw2Wav(rawDirName, wavDirName);             
-       } else {
-         // check if the user has provided a raw directory  
-         if( !userRawDirName.equals("") ) {
-           File userRawDir  = new File(userRawDirName);
-         
-           // check if user provided raw dir contains files
-           if( existWithFiles(userRawDirName) ) {
-             // copy the user provided raw directory to hts/data/raw/
-             System.out.println("Copying files from: " + userRawDirName + "  to: " + rawDirName);
-             FileUtils.copyFolder(userRawDirName, rawDirName);
-           
-             // the raw files should be the same as in wav file, if wav file empty convert from raw --> wav
-             if( !existWithFiles(wavDirName))
-               convertRaw2Wav(rawDirName, wavDirName);    
-            
-             raw = true;
-           } else
-             System.out.println("User provided raw directory: " + userRawDirName + " does not exist or does not contain files\n");
-         }
-         // if we still do not have raw files...
-         // then there must be a wav directory, check that it contains files, if so convert wav --> raw
-         if(!raw) {
-           System.out.println("Checking if " + wavDirName + " contains files");
-           if( existWithFiles(wavDirName) ){
-             convertWav2Raw(wavDirName, rawDirName);            
-             raw = true;
-           } else {
-             System.out.println("There are no wav files in " + wavDirName);  
-           }
-         }
-       } 
-       
-       // 2.2 check text files:
-       if( existWithFiles(textDirName) ) {
-         text = true;
-       } else if( existWithFiles(uttsDirName) ) {
-           convertUtt2Text(uttsDirName, textDirName);
-           text = true;             
-       } else {   
-         // check if the user has provided a utterance directory
-         String userUttDirName = getProp(USERUTTDIR);
-         if( !userUttDirName.equals("") ) {
-           // check if user provided utt dir contains files
-           if( existWithFiles(userUttDirName) ) {
-             // convert utt --> text (transcriptions festival format --> MARY format)
-             convertUtt2Text(userUttDirName, textDirName);
-             text = true;
-           } else
-             System.out.println("User provided utterance directory: " + userUttDirName + " does not exist or does not contain files\n");
-         } else {
-           System.out.println("\nThere are no text files in " + textDirName);
-           text = false;
-         }
-       }
-       
-       if( raw && text ){
-         System.out.println("\nHMMVoiceDataPreparation finished:\n" +
-                "HTS speaker independent scripts copied in current voice building directory --> hts\n" +
-                "wav/raw and text directories in place.");  
-         return true;
-       }
-       else
-         return false;
-       
-       } else {  // ADAPTSCRIPTS == true
-           
-           // Here it is checked that the raw files are in data/raw, wav, phonelab and phonefeatures must be 
-           // provided by the user...
-           // 1. copy from $MARY_TTS/lib/external/hts-adapt directory in the voice building directory
-           String sourceFolder = marybase + sep + "lib" + sep + "external" + sep + "hts-adapt";
-           String htsFolder = voiceDir + sep + "hts";          
-           FileUtils.copyFolderRecursive(sourceFolder, htsFolder, false);
-           
-           
-           // 2. check as before that wav, raw and text directories exist and are in the correct place     
-           //System.out.println("\nChecking raw directory for running HTS adaptive training scripts...");
+	/**
+	 * Do the computations required by this component.
+	 * 
+	 * @return true on success, false on failure
+	 */
+	public boolean compute() throws Exception {
+		boolean raw = false;
+		boolean text = false;
+		boolean wav = false;
+		marybase = db.getProp(DatabaseLayout.MARYBASE);
+		voiceDir = db.getProp(DatabaseLayout.ROOTDIR);
+		soxPath = db.getExternal(DatabaseLayout.SOXPATH);
+		sep = System.getProperty("file.separator");
+		dataDir = voiceDir + "hts" + sep + "data" + sep;
+		scriptsDir = dataDir + "scripts" + sep;
 
-           File dirSpeakersRaw = new File(dataDir + "/raw");
-           
-           String[] speakers;
-           if(dirSpeakersRaw.exists() && dirSpeakersRaw.list().length > 0){ 
-               speakers = dirSpeakersRaw.list();
-               for(int i=0; i<speakers.length; i++){
-                 File dirSpeakerRaw = new File(dataDir + "/raw/" + speakers[i]);
-                 if(dirSpeakerRaw.exists() && dirSpeakerRaw.list().length > 0  ){
-                   raw = true;
-                 } else {
-                   System.out.println("Error: directory " + voiceDir + "/raw/" + speakers[i] + " does not contain files." );
-                   raw = false;
-                   break;
-                 }
-               }
-           } else {            
-               System.out.println("Error: directory " + voiceDir + "/raw does not contain files." );
-               raw = false;
-           }
-           
-           if( raw){
-               System.out.println("\nHMMVoiceDataPreparation finished:\n" +
-                      "HTS adapt scripts copied in current voice building directory --> hts\n" +
-                      "raw directory in place.");  
-               return true;
-             }
-             else
-               return false;
-           
-       }
-       
-    }
-    
-    
-    /**
-     * Check the paths of all the necessary external programs
-     * @return true if all the paths are defined
-     */
-    private boolean checkExternalPaths() throws Exception{
-        
-        boolean result = true;
-        
-        if ( db.getExternal(DatabaseLayout.AWKPATH) == null ){
-          System.out.println("  *Missing path for awk"); 
-          result = false;
-        }
-        if (db.getExternal(DatabaseLayout.PERLPATH) == null){
-            System.out.println("  *Missing path for perl"); 
-            result = false;
-        }        
-        if (db.getExternal(DatabaseLayout.BCPATH) == null){
-            System.out.println("  *Missing path for bc"); 
-            result = false;
-        }
-        if (db.getExternal(DatabaseLayout.TCLPATH) == null){
-            System.out.println("  *Missing path for tclsh"); 
-            result = false;
-        }
-        if (db.getExternal(DatabaseLayout.SOXPATH) == null){
-            System.out.println("  *Missing path for sox"); 
-            result = false;
-        }
-        if(db.getExternal(DatabaseLayout.HTSPATH) == null){
-            System.out.println("  *Missing path for hts/htk"); 
-            result = false;
-        }
-        if(db.getExternal(DatabaseLayout.HTSENGINEPATH) == null){
-            System.out.println("  *Missing path for hts_engine"); 
-            result = false;
-        }
-        if(db.getExternal(DatabaseLayout.SPTKPATH) == null){
-            System.out.println("  *Missing path for sptk"); 
-            result = false;
-        }
-        if(db.getExternal(DatabaseLayout.EHMMPATH) == null){
-            System.out.println("  *Missing path for ehmm"); 
-            result = false;
-        }
-       
-        if(!result)
-          System.out.println("Please run MARYBASE/lib/external/check_install_external_programs.sh and check/install the missing programs");
-        else
-          System.out.println("Paths for all external programs are defined.");  
-        
-        return result;
-    }
-    
-    /**
-     * Checks if the directory exist and has files
-     * @param dir
-     * @return
-     */
-    private boolean existWithFiles(String dirName) {
-      File dir = new File(dirName);
-      if( dir.exists() && dir.list().length>0 )
-        return true;
-      else
-        return false;
-    }
+		// For both speaker indep. or adapt scripts the programs are the same
+		// check again that all the external necessary programs are installed.
+		System.out.println("\nHMMVoiceDataPreparation:\nChecking paths of external programs");
+		if (!checkExternalPaths())
+			return false;
+
+		if (getProp(ADAPTSCRIPTS).contentEquals("false")) {
+
+			// 1. copy from $MARY_TTS/lib/external/hts directory in the voice building directory
+			String sourceFolder = marybase + sep + "lib" + sep + "external" + sep + "hts";
+			String htsFolder = voiceDir + sep + "hts";
+			FileUtils.copyFolderRecursive(sourceFolder, htsFolder, false);
+
+			// 2. check as before that wav, raw and text directories exist and are in the correct place
+			System.out
+					.println("\nChecking wav/raw and text directories and files for running HTS speaker independent training scripts...");
+
+			// default locations of directories:
+			String wavDirName = voiceDir + "wav";
+			String textDirName = voiceDir + "text";
+			String rawDirName = dataDir + "raw";
+			String uttsDirName = dataDir + "utts";
+
+			// 2.1 check raw and wav files:
+			String userRawDirName = getProp(USERRAWDIR);
+			if (existWithFiles(rawDirName)) {
+				raw = true;
+				// the raw files should be the same as in wav file, if wav file empty convert from raw --> wav
+				if (!existWithFiles(wavDirName))
+					convertRaw2Wav(rawDirName, wavDirName);
+			} else {
+				// check if the user has provided a raw directory
+				if (!userRawDirName.equals("")) {
+					File userRawDir = new File(userRawDirName);
+
+					// check if user provided raw dir contains files
+					if (existWithFiles(userRawDirName)) {
+						// copy the user provided raw directory to hts/data/raw/
+						System.out.println("Copying files from: " + userRawDirName + "  to: " + rawDirName);
+						FileUtils.copyFolder(userRawDirName, rawDirName);
+
+						// the raw files should be the same as in wav file, if wav file empty convert from raw --> wav
+						if (!existWithFiles(wavDirName))
+							convertRaw2Wav(rawDirName, wavDirName);
+
+						raw = true;
+					} else
+						System.out.println("User provided raw directory: " + userRawDirName
+								+ " does not exist or does not contain files\n");
+				}
+				// if we still do not have raw files...
+				// then there must be a wav directory, check that it contains files, if so convert wav --> raw
+				if (!raw) {
+					System.out.println("Checking if " + wavDirName + " contains files");
+					if (existWithFiles(wavDirName)) {
+						convertWav2Raw(wavDirName, rawDirName);
+						raw = true;
+					} else {
+						System.out.println("There are no wav files in " + wavDirName);
+					}
+				}
+			}
+
+			// 2.2 check text files:
+			if (existWithFiles(textDirName)) {
+				text = true;
+			} else if (existWithFiles(uttsDirName)) {
+				convertUtt2Text(uttsDirName, textDirName);
+				text = true;
+			} else {
+				// check if the user has provided a utterance directory
+				String userUttDirName = getProp(USERUTTDIR);
+				if (!userUttDirName.equals("")) {
+					// check if user provided utt dir contains files
+					if (existWithFiles(userUttDirName)) {
+						// convert utt --> text (transcriptions festival format --> MARY format)
+						convertUtt2Text(userUttDirName, textDirName);
+						text = true;
+					} else
+						System.out.println("User provided utterance directory: " + userUttDirName
+								+ " does not exist or does not contain files\n");
+				} else {
+					System.out.println("\nThere are no text files in " + textDirName);
+					text = false;
+				}
+			}
+
+			if (raw && text) {
+				System.out.println("\nHMMVoiceDataPreparation finished:\n"
+						+ "HTS speaker independent scripts copied in current voice building directory --> hts\n"
+						+ "wav/raw and text directories in place.");
+				return true;
+			} else
+				return false;
+
+		} else { // ADAPTSCRIPTS == true
+
+			// Here it is checked that the raw files are in data/raw, wav, phonelab and phonefeatures must be
+			// provided by the user...
+			// 1. copy from $MARY_TTS/lib/external/hts-adapt directory in the voice building directory
+			String sourceFolder = marybase + sep + "lib" + sep + "external" + sep + "hts-adapt";
+			String htsFolder = voiceDir + sep + "hts";
+			FileUtils.copyFolderRecursive(sourceFolder, htsFolder, false);
+
+			// 2. check as before that wav, raw and text directories exist and are in the correct place
+			// System.out.println("\nChecking raw directory for running HTS adaptive training scripts...");
+
+			File dirSpeakersRaw = new File(dataDir + "/raw");
+
+			String[] speakers;
+			if (dirSpeakersRaw.exists() && dirSpeakersRaw.list().length > 0) {
+				speakers = dirSpeakersRaw.list();
+				for (int i = 0; i < speakers.length; i++) {
+					File dirSpeakerRaw = new File(dataDir + "/raw/" + speakers[i]);
+					if (dirSpeakerRaw.exists() && dirSpeakerRaw.list().length > 0) {
+						raw = true;
+					} else {
+						System.out.println("Error: directory " + voiceDir + "/raw/" + speakers[i] + " does not contain files.");
+						raw = false;
+						break;
+					}
+				}
+			} else {
+				System.out.println("Error: directory " + voiceDir + "/raw does not contain files.");
+				raw = false;
+			}
+
+			if (raw) {
+				System.out.println("\nHMMVoiceDataPreparation finished:\n"
+						+ "HTS adapt scripts copied in current voice building directory --> hts\n" + "raw directory in place.");
+				return true;
+			} else
+				return false;
+
+		}
+
+	}
+
+	/**
+	 * Check the paths of all the necessary external programs
+	 * 
+	 * @return true if all the paths are defined
+	 */
+	private boolean checkExternalPaths() throws Exception {
+
+		boolean result = true;
+
+		if (db.getExternal(DatabaseLayout.AWKPATH) == null) {
+			System.out.println("  *Missing path for awk");
+			result = false;
+		}
+		if (db.getExternal(DatabaseLayout.PERLPATH) == null) {
+			System.out.println("  *Missing path for perl");
+			result = false;
+		}
+		if (db.getExternal(DatabaseLayout.BCPATH) == null) {
+			System.out.println("  *Missing path for bc");
+			result = false;
+		}
+		if (db.getExternal(DatabaseLayout.TCLPATH) == null) {
+			System.out.println("  *Missing path for tclsh");
+			result = false;
+		}
+		if (db.getExternal(DatabaseLayout.SOXPATH) == null) {
+			System.out.println("  *Missing path for sox");
+			result = false;
+		}
+		if (db.getExternal(DatabaseLayout.HTSPATH) == null) {
+			System.out.println("  *Missing path for hts/htk");
+			result = false;
+		}
+		if (db.getExternal(DatabaseLayout.HTSENGINEPATH) == null) {
+			System.out.println("  *Missing path for hts_engine");
+			result = false;
+		}
+		if (db.getExternal(DatabaseLayout.SPTKPATH) == null) {
+			System.out.println("  *Missing path for sptk");
+			result = false;
+		}
+		if (db.getExternal(DatabaseLayout.EHMMPATH) == null) {
+			System.out.println("  *Missing path for ehmm");
+			result = false;
+		}
+
+		if (!result)
+			System.out
+					.println("Please run MARYBASE/lib/external/check_install_external_programs.sh and check/install the missing programs");
+		else
+			System.out.println("Paths for all external programs are defined.");
+
+		return result;
+	}
+
+	/**
+	 * Checks if the directory exist and has files
+	 * 
+	 * @param dir
+	 * @return
+	 */
+	private boolean existWithFiles(String dirName) {
+		File dir = new File(dirName);
+		if (dir.exists() && dir.list().length > 0)
+			return true;
+		else
+			return false;
+	}
+
+	private void convertRaw2Wav(String rawDirName, String wavDirName) {
+		String Fs = db.getProperty(DatabaseLayout.SAMPLINGRATE);
+		String cmdLine;
+		String raw2wavCmd = scriptsDir + "raw2wav.sh";
+		System.out.println("Converting raw files to wav from: " + rawDirName + "  to: " + wavDirName);
+		File wavDir = new File(wavDirName);
+		if (!wavDir.exists())
+			wavDir.mkdir();
+		cmdLine = "chmod +x " + raw2wavCmd;
+		General.launchProc(cmdLine, "raw2wav", voiceDir);
+		cmdLine = raw2wavCmd + " " + soxPath + " " + rawDirName + " " + wavDirName + " " + Fs;
+		General.launchProc(cmdLine, "raw2wav", voiceDir);
+	}
+
+	private void convertWav2Raw(String wavDirName, String rawDirName) {
+		String cmdLine;
+		String wav2rawCmd = scriptsDir + "wav2raw.sh";
+		System.out.println("Converting wav files to raw from: " + wavDirName + "  to: " + rawDirName);
+		File rawDir = new File(rawDirName);
+		if (!rawDir.exists())
+			rawDir.mkdir();
+		cmdLine = "chmod +x " + wav2rawCmd;
+		General.launchProc(cmdLine, "wav2raw", voiceDir);
+		cmdLine = wav2rawCmd + " " + soxPath + " " + wavDirName + " " + rawDirName;
+		General.launchProc(cmdLine, "wav2raw", voiceDir);
+	}
+
+	private void convertUtt2Text(String userUttDirName, String textDirName) {
+		String cmdLine;
+		String utt2transCmd = scriptsDir + "utt2trans.sh"; // festival to mary format
+		System.out.println("\nConverting transcription files (festival format) to text from: " + userUttDirName + "  to: "
+				+ textDirName);
+		File textDir = new File(textDirName);
+		if (!textDir.exists())
+			textDir.mkdir();
+		cmdLine = "chmod +x " + utt2transCmd;
+		General.launchProc(cmdLine, "utt2trans", voiceDir);
+		cmdLine = utt2transCmd + " " + userUttDirName + " " + textDirName;
+		General.launchProc(cmdLine, "utt2trans", voiceDir);
+
+	}
+
+	/**
+	 * Provide the progress of computation, in percent, or -1 if that feature is not implemented.
+	 * 
+	 * @return -1 if not implemented, or an integer between 0 and 100.
+	 */
+	public int getProgress() {
+		return -1;
+	}
 
-    private void convertRaw2Wav(String rawDirName, String wavDirName) {
-        String Fs = db.getProperty(DatabaseLayout.SAMPLINGRATE);
-        String cmdLine;
-        String raw2wavCmd   = scriptsDir + "raw2wav.sh";
-        System.out.println("Converting raw files to wav from: " + rawDirName + "  to: " + wavDirName);
-        File wavDir = new File(wavDirName);
-        if(!wavDir.exists())
-            wavDir.mkdir();
-        cmdLine = "chmod +x " + raw2wavCmd;
-        General.launchProc(cmdLine, "raw2wav", voiceDir);
-        cmdLine = raw2wavCmd + " " + soxPath + " " + rawDirName + " " + wavDirName + " " + Fs;
-        General.launchProc(cmdLine, "raw2wav", voiceDir);  
-    }
-    
-    private void convertWav2Raw(String wavDirName, String rawDirName) {
-        String cmdLine;
-        String wav2rawCmd   = scriptsDir + "wav2raw.sh";
-        System.out.println("Converting wav files to raw from: " + wavDirName + "  to: " + rawDirName);
-        File rawDir  = new File(rawDirName);
-        if(!rawDir.exists())
-          rawDir.mkdir();  
-        cmdLine = "chmod +x " + wav2rawCmd;
-        General.launchProc(cmdLine, "wav2raw", voiceDir);
-        cmdLine = wav2rawCmd + " " + soxPath + " " + wavDirName + " " + rawDirName ;
-        General.launchProc(cmdLine, "wav2raw", voiceDir);  
-    }
-    
-    private void convertUtt2Text(String userUttDirName, String textDirName) {
-        String cmdLine;
-        String utt2transCmd = scriptsDir + "utt2trans.sh";  // festival to mary format
-        System.out.println("\nConverting transcription files (festival format) to text from: " + userUttDirName + "  to: " + textDirName);
-        File textDir = new File(textDirName);
-        if(!textDir.exists())
-          textDir.mkdir();
-        cmdLine = "chmod +x " + utt2transCmd;
-        General.launchProc(cmdLine, "utt2trans", voiceDir);
-        cmdLine = utt2transCmd + " " + userUttDirName + " " + textDirName;
-        General.launchProc(cmdLine, "utt2trans", voiceDir);
- 
-    }
-   
-    
-    /**
-     * Provide the progress of computation, in percent, or -1 if
-     * that feature is not implemented.
-     * @return -1 if not implemented, or an integer between 0 and 100.
-     */
-    public int getProgress(){
-        return -1;
-    }
-    
-    
-    
-    
-    
 }
-

marytts-builder/src/main/java/marytts/tools/voiceimport/HMMVoiceFeatureSelection.java

@@ -41,207 +41,190 @@ import javax.swing.JFrame;
 import javax.swing.JPanel;
 import javax.swing.JScrollPane;
 
-
 public class HMMVoiceFeatureSelection extends VoiceImportComponent {
-    
-    //protected File hmmFeatureFile11;
-    protected String features;
-    protected String locale;
-    protected boolean success = true;
-    
-    protected DatabaseLayout db = null;
-    
-    public String HMMFEATUREFILE = "HMMVoiceFeatureSelection.hmmFeatureFile";
-    public String FEATUREFILE = "HMMVoiceFeatureSelection.featureFile";
-        
-    public String getName(){
-        return "HMMVoiceFeatureSelection";
-    }
-    
-    
-    @Override
-    protected void initialiseComp()
-    {      
-        locale = db.getProp(db.LOCALE);           
-    }
-     
-     public SortedMap<String,String> getDefaultProps(DatabaseLayout theDb){
-         this.db = theDb;
-         if (props == null){
-             props = new TreeMap<String, String>();
-             props.put(HMMFEATUREFILE, db.getProp(DatabaseLayout.CONFIGDIR) + "hmmFeatures.txt");
-             props.put(FEATUREFILE, db.getProp(DatabaseLayout.CONFIGDIR) + "features.txt");
-             
-         } 
-         return props;
-     }
-     
-     protected void setupHelp() {
-         props2Help = new TreeMap<String, String>();
-         props2Help.put(HMMFEATUREFILE, "file containing the extra features, appart from phone and phonological, that will be used to train the HMMs." 
-                 +" The file will be created by reading mary/features.txt, normally hmmFeatures.txt is a subset of features.txt. " +
-                  " Delete the features that will no be used to train the HMMs. When running this program " +
-                  " a small set will be presented on top, (pos_in_syl, syl_break, prev_syl_break, position_type) separated by an empty line, " +
-                  " if you are not sure about using other features, use just these and delete the others.");
-         props2Help.put(FEATUREFILE, "file containing the list of features used to create the phonefeatures, this file should had been created " +
-                "with the FeatureSelection component.");
-     }
-     
-         
-     protected void saveFeatures(String newFeatures)
-     {
-         File hmmFeatureFile = new File(getProp(HMMFEATUREFILE));
-         System.out.println("Saving extra features for training HMMs to "+ hmmFeatureFile.getAbsolutePath());
-         features = newFeatures;
-         /*if (!features.contains(PhoneUnitFeatureComputer.PHONEFEATURE)) {
-             JOptionPane.showMessageDialog(null, "The features '"+PhoneUnitFeatureComputer.PHONEFEATURE
-                     +"' is not present.\nThis will lead to problems in the further processing.",
-                     "Important feature missing", JOptionPane.WARNING_MESSAGE);
-         }*/
-         try {
-             PrintWriter pw = new PrintWriter(hmmFeatureFile, "UTF-8");
-             pw.println(features);
-             pw.close();
-         } catch (IOException e) {
-             System.err.println("Cannot save features:");
-             e.printStackTrace();
-             success = false;
-         }
-     }
-     
-          
-     protected void setSuccess(boolean val)
-     {
-         success = val;
-     }
 
-    public boolean compute() throws IOException
-    {
-        features = loadFeatureList();
-        if (!features.contentEquals("")) {
-            features = features.replaceAll(" ", "\n");
-            features = features.replaceFirst("unit_duration", "");
-            features = features.replaceFirst("unit_logf0delta", "");
-            features = features.replaceFirst("unit_logf0", "");
-        }
+	// protected File hmmFeatureFile11;
+	protected String features;
+	protected String locale;
+	protected boolean success = true;
 
-        final JFrame frame = new JFrame("Extra features for training HMMs");
-        GridBagLayout gridBagLayout = new GridBagLayout();
-        GridBagConstraints gridC = new GridBagConstraints();
-        frame.getContentPane().setLayout( gridBagLayout );
-        
-        final JEditorPane editPane = new JEditorPane();
-        editPane.setPreferredSize(new Dimension(500, 500));
-        editPane.setText(features);        
-        
-        JButton saveButton = new JButton("Save");
-        saveButton.addActionListener(new ActionListener() {
-           public void actionPerformed(ActionEvent e) {                
-               setSuccess(true);
-               frame.setVisible(false);
-            }
-        });
-        JButton cancelButton = new JButton("Cancel");
-        cancelButton.addActionListener(new ActionListener() {
-            public void actionPerformed(ActionEvent e) {
-                setSuccess(false);
-                frame.setVisible(false);
-            }
-        });
-        
-        gridC.gridx = 0;
-        gridC.gridy = 0;
-        // resize scroll pane:
-        gridC.weightx = 1;
-        gridC.weighty = 1;
-        gridC.fill = GridBagConstraints.HORIZONTAL;
-        JScrollPane scrollPane = new JScrollPane(editPane);
-        scrollPane.setPreferredSize(editPane.getPreferredSize());
-        gridBagLayout.setConstraints( scrollPane, gridC );
-        frame.getContentPane().add(scrollPane);
-        gridC.gridy = 1;
-        // do not resize buttons:
-        gridC.weightx = 0;
-        gridC.weighty = 0;
-        JPanel buttonPanel = new JPanel();
-        buttonPanel.setLayout(new FlowLayout());
-        buttonPanel.add(saveButton);
-        buttonPanel.add(cancelButton);
-        gridBagLayout.setConstraints( buttonPanel, gridC );
-        frame.getContentPane().add(buttonPanel);
-        frame.pack();
-        frame.setVisible(true);
-        
-        do {
-            try {
-                Thread.sleep(10); 
-            } catch (InterruptedException e) {}
-        } while (frame.isVisible());
-        frame.dispose();
+	protected DatabaseLayout db = null;
 
-        if (success) {
-            try{
-                saveFeatures(editPane.getText());                
-            } catch (Exception ex){
-                ex.printStackTrace();
-                throw new Error("Error defining replacements");
-            }
-        }
+	public String HMMFEATUREFILE = "HMMVoiceFeatureSelection.hmmFeatureFile";
+	public String FEATUREFILE = "HMMVoiceFeatureSelection.featureFile";
 
-        return success;
-    }
+	public String getName() {
+		return "HMMVoiceFeatureSelection";
+	}
 
+	@Override
+	protected void initialiseComp() {
+		locale = db.getProp(db.LOCALE);
+	}
 
-    private String loadFeatureList() throws IOException
-    {
-        String featureList = "";
-        String recommendedFeatureList = "";
-        Scanner feaList = null;
-        try {
-          feaList = new Scanner(new BufferedReader(new FileReader(getProp(FEATUREFILE))));
-          String line;
-          while (feaList.hasNext()) {
-            line = feaList.nextLine();
-            // Exclude phone and phonological, those are by default used in makeLabes and makeQuestions
-            // Also exclude the halfphone features not used in HMM voices
-            if(! (line.contains("_vc") ||
-                  line.contains("_vlng") ||
-                  line.contains("_vheight") ||
-                  line.contains("_vfront") ||
-                  line.contains("_vrnd") ||
-                  line.contains("_ctype") ||
-                  line.contains("_cplace") ||
-                  line.contains("_cvox") ||
-                  line.contains("_phone") || 
-                  line.contains("ph_") ||
-                  line.contains("halfphone_") ||
-                  line.contentEquals("phone") )  ) {
-                               
-                featureList += line + " ";
-            }
-                
-          }
-          if (feaList != null) { 
-            feaList.close();
-          }       
-        } catch (FileNotFoundException e) {            
-             throw new FileNotFoundException();
-        }              
-        return featureList;
+	public SortedMap<String, String> getDefaultProps(DatabaseLayout theDb) {
+		this.db = theDb;
+		if (props == null) {
+			props = new TreeMap<String, String>();
+			props.put(HMMFEATUREFILE, db.getProp(DatabaseLayout.CONFIGDIR) + "hmmFeatures.txt");
+			props.put(FEATUREFILE, db.getProp(DatabaseLayout.CONFIGDIR) + "features.txt");
 
-    }
+		}
+		return props;
+	}
 
-    
-    /**
-     * Provide the progress of computation, in percent, or -1 if
-     * that feature is not implemented.
-     * @return -1 if not implemented, or an integer between 0 and 100.
-     */
-    public int getProgress()
-    {
-        return -1;
-    }
-    
-    
+	protected void setupHelp() {
+		props2Help = new TreeMap<String, String>();
+		props2Help
+				.put(HMMFEATUREFILE,
+						"file containing the extra features, appart from phone and phonological, that will be used to train the HMMs."
+								+ " The file will be created by reading mary/features.txt, normally hmmFeatures.txt is a subset of features.txt. "
+								+ " Delete the features that will no be used to train the HMMs. When running this program "
+								+ " a small set will be presented on top, (pos_in_syl, syl_break, prev_syl_break, position_type) separated by an empty line, "
+								+ " if you are not sure about using other features, use just these and delete the others.");
+		props2Help.put(FEATUREFILE,
+				"file containing the list of features used to create the phonefeatures, this file should had been created "
+						+ "with the FeatureSelection component.");
+	}
+
+	protected void saveFeatures(String newFeatures) {
+		File hmmFeatureFile = new File(getProp(HMMFEATUREFILE));
+		System.out.println("Saving extra features for training HMMs to " + hmmFeatureFile.getAbsolutePath());
+		features = newFeatures;
+		/*
+		 * if (!features.contains(PhoneUnitFeatureComputer.PHONEFEATURE)) { JOptionPane.showMessageDialog(null,
+		 * "The features '"+PhoneUnitFeatureComputer.PHONEFEATURE
+		 * +"' is not present.\nThis will lead to problems in the further processing.", "Important feature missing",
+		 * JOptionPane.WARNING_MESSAGE); }
+		 */
+		try {
+			PrintWriter pw = new PrintWriter(hmmFeatureFile, "UTF-8");
+			pw.println(features);
+			pw.close();
+		} catch (IOException e) {
+			System.err.println("Cannot save features:");
+			e.printStackTrace();
+			success = false;
+		}
+	}
+
+	protected void setSuccess(boolean val) {
+		success = val;
+	}
+
+	public boolean compute() throws IOException {
+		features = loadFeatureList();
+		if (!features.contentEquals("")) {
+			features = features.replaceAll(" ", "\n");
+			features = features.replaceFirst("unit_duration", "");
+			features = features.replaceFirst("unit_logf0delta", "");
+			features = features.replaceFirst("unit_logf0", "");
+		}
+
+		final JFrame frame = new JFrame("Extra features for training HMMs");
+		GridBagLayout gridBagLayout = new GridBagLayout();
+		GridBagConstraints gridC = new GridBagConstraints();
+		frame.getContentPane().setLayout(gridBagLayout);
+
+		final JEditorPane editPane = new JEditorPane();
+		editPane.setPreferredSize(new Dimension(500, 500));
+		editPane.setText(features);
+
+		JButton saveButton = new JButton("Save");
+		saveButton.addActionListener(new ActionListener() {
+			public void actionPerformed(ActionEvent e) {
+				setSuccess(true);
+				frame.setVisible(false);
+			}
+		});
+		JButton cancelButton = new JButton("Cancel");
+		cancelButton.addActionListener(new ActionListener() {
+			public void actionPerformed(ActionEvent e) {
+				setSuccess(false);
+				frame.setVisible(false);
+			}
+		});
+
+		gridC.gridx = 0;
+		gridC.gridy = 0;
+		// resize scroll pane:
+		gridC.weightx = 1;
+		gridC.weighty = 1;
+		gridC.fill = GridBagConstraints.HORIZONTAL;
+		JScrollPane scrollPane = new JScrollPane(editPane);
+		scrollPane.setPreferredSize(editPane.getPreferredSize());
+		gridBagLayout.setConstraints(scrollPane, gridC);
+		frame.getContentPane().add(scrollPane);
+		gridC.gridy = 1;
+		// do not resize buttons:
+		gridC.weightx = 0;
+		gridC.weighty = 0;
+		JPanel buttonPanel = new JPanel();
+		buttonPanel.setLayout(new FlowLayout());
+		buttonPanel.add(saveButton);
+		buttonPanel.add(cancelButton);
+		gridBagLayout.setConstraints(buttonPanel, gridC);
+		frame.getContentPane().add(buttonPanel);
+		frame.pack();
+		frame.setVisible(true);
+
+		do {
+			try {
+				Thread.sleep(10);
+			} catch (InterruptedException e) {
+			}
+		} while (frame.isVisible());
+		frame.dispose();
+
+		if (success) {
+			try {
+				saveFeatures(editPane.getText());
+			} catch (Exception ex) {
+				ex.printStackTrace();
+				throw new Error("Error defining replacements");
+			}
+		}
+
+		return success;
+	}
+
+	private String loadFeatureList() throws IOException {
+		String featureList = "";
+		String recommendedFeatureList = "";
+		Scanner feaList = null;
+		try {
+			feaList = new Scanner(new BufferedReader(new FileReader(getProp(FEATUREFILE))));
+			String line;
+			while (feaList.hasNext()) {
+				line = feaList.nextLine();
+				// Exclude phone and phonological, those are by default used in makeLabes and makeQuestions
+				// Also exclude the halfphone features not used in HMM voices
+				if (!(line.contains("_vc") || line.contains("_vlng") || line.contains("_vheight") || line.contains("_vfront")
+						|| line.contains("_vrnd") || line.contains("_ctype") || line.contains("_cplace")
+						|| line.contains("_cvox") || line.contains("_phone") || line.contains("ph_")
+						|| line.contains("halfphone_") || line.contentEquals("phone"))) {
+
+					featureList += line + " ";
+				}
+
+			}
+			if (feaList != null) {
+				feaList.close();
+			}
+		} catch (FileNotFoundException e) {
+			throw new FileNotFoundException();
+		}
+		return featureList;
+
+	}
+
+	/**
+	 * Provide the progress of computation, in percent, or -1 if that feature is not implemented.
+	 * 
+	 * @return -1 if not implemented, or an integer between 0 and 100.
+	 */
+	public int getProgress() {
+		return -1;
+	}
 
 }

marytts-builder/src/main/java/marytts/tools/voiceimport/HMMVoiceMakeVoice.java

@@ -74,136 +74,133 @@ import java.util.Locale;
 import java.util.SortedMap;
 import java.util.TreeMap;
 
-public class HMMVoiceMakeVoice extends VoiceImportComponent{
-    
-    private DatabaseLayout db;
-    private String name = "HMMVoiceMakeVoice";
+public class HMMVoiceMakeVoice extends VoiceImportComponent {
+
+	private DatabaseLayout db;
+	private String name = "HMMVoiceMakeVoice";
+
+	public String getName() {
+		return name;
+	}
+
+	/**
+	 * Get the map of properties2values containing the default values
+	 * 
+	 * @return map of props2values
+	 */
+	public SortedMap<String, String> getDefaultProps(DatabaseLayout db) {
+		this.db = db;
+		if (props == null) {
+			props = new TreeMap<String, String>();
+			// props.put("command:", "perl hts/scripts/Training.pl hts/scripts/Config.pm");
+		}
+		return props;
+	}
+
+	protected void setupHelp() {
+		props2Help = new TreeMap<String, String>();
+		/*
+		 * props2Help.put("command:", "This command can be executed on the command line by: " +
+		 * "first going to the directory: /your_voice/hts/ " + "and in this directory execute: " + "\"make voice\" " +
+		 * "a log file is created in this directory.");
+		 */
+
+	}
+
+	/**
+	 * Do the computations required by this component.
+	 * 
+	 * @return true on success, false on failure
+	 */
+	public boolean compute() throws Exception {
+
+		String cmdLine;
+		String voicedir = db.getProp(db.ROOTDIR);
+
+		/* Run: perl hts/scripts/Training.pl hts/scripts/Config.pm (It can take several hours...) */
+		cmdLine = db.getExternal(db.PERLPATH) + "/perl " + voicedir + "hts/scripts/Training.pl " + voicedir
+				+ "hts/scripts/Config.pm";
+		launchProcWithLogFile(cmdLine, "", voicedir);
+
+		return true;
+	}
+
+	/**
+	 * A general process launcher for the various tasks (copied from ESTCaller.java)
+	 * 
+	 * @param cmdLine
+	 *            the command line to be launched.
+	 * @param task
+	 *            a task tag for error messages, such as "Pitchmarks" or "LPC".
+	 * @param the
+	 *            basename of the file currently processed, for error messages.
+	 */
+	private void launchProcWithLogFile(String cmdLine, String task, String voicedir) {
+
+		Process proc = null;
+		BufferedReader procStdout = null;
+		String line = null;
+
+		Date today;
+		String output;
+		SimpleDateFormat formatter;
+		formatter = new SimpleDateFormat("yyyy.MM.dd-H:mm:ss", new Locale("en", "US"));
+		today = new Date();
+		output = formatter.format(today);
+		String logFile = voicedir + "hts/log-" + output;
+
+		System.out.println("\nRunning: " + cmdLine);
+		System.out.println("\nThe training procedure can take several hours...");
+		System.out.println("Detailed information about the training status can be found in the logfile:\n  " + logFile);
+		System.out.println("\nTraining voice: " + db.getProp(db.VOICENAME));
+		System.out.println("The following is general information about execution of training steps:");
+		// String[] cmd = null; // Java 5.0 compliant code
+
+		try {
+			FileWriter log = new FileWriter(logFile);
+			/* Java 5.0 compliant code below. */
+			/* Hook the command line to the process builder: */
+			ProcessBuilder pb = new ProcessBuilder(cmdLine.split(" "));
+			pb.directory(new File(db.getProp(db.ROOTDIR)));
+			pb.redirectErrorStream(true);
+			/* Launch the process: */
+			proc = pb.start();
+
+			/* Collect process's combined stdout & stderr send it to System.out: */
+			procStdout = new BufferedReader(new InputStreamReader(proc.getInputStream()));
+			while ((line = procStdout.readLine()) != null) {
+				if (line.contains("Start "))
+					System.out.println("\nStep: " + line);
+				log.write(line + "\n");
+			}
+
+			/* Wait and check the exit value */
+			proc.waitFor();
+			if (proc.exitValue() != 0) {
+				BufferedReader errReader = new BufferedReader(new InputStreamReader(proc.getErrorStream()));
+				while ((line = errReader.readLine()) != null) {
+					System.err.println("ERR> " + line);
+				}
+				errReader.close();
+				throw new RuntimeException(task + " computation failed on file [" + voicedir + "]!\n" + "Command line was: ["
+						+ cmdLine + "].");
+			}
+			log.close();
+		} catch (IOException e) {
+			throw new RuntimeException(task + " computation provoked an IOException on file [" + voicedir + "].", e);
+		} catch (InterruptedException e) {
+			throw new RuntimeException(task + " computation interrupted on file [" + voicedir + "].", e);
+		}
+
+	}
+
+	/**
+	 * Provide the progress of computation, in percent, or -1 if that feature is not implemented.
+	 * 
+	 * @return -1 if not implemented, or an integer between 0 and 100.
+	 */
+	public int getProgress() {
+		return -1;
+	}
 
-    public String getName(){
-        return name;
-    }
-    
-    /**
-     * Get the map of properties2values
-     * containing the default values
-     * @return map of props2values
-     */
-    public SortedMap<String,String> getDefaultProps(DatabaseLayout db){
-        this.db = db;
-       if (props == null){
-           props = new TreeMap<String,String>();
-           //props.put("command:", "perl hts/scripts/Training.pl hts/scripts/Config.pm");
-       }
-       return props;
-       }
-    
-    protected void setupHelp(){
-        props2Help = new TreeMap<String,String>();
-        /*props2Help.put("command:", "This command can be executed on the command line by: " +
-        		"first going to the directory: /your_voice/hts/ " +
-        		"and in this directory execute: " +
-        		"\"make voice\" " +
-        		"a log file is created in this directory.");
-       */
-        		
-    }
- 
-    
-    /**
-     * Do the computations required by this component.
-     * 
-     * @return true on success, false on failure
-     */
-    public boolean compute() throws Exception{
-      
-      String cmdLine;
-      String voicedir = db.getProp(db.ROOTDIR);
-            
-      /* Run: perl hts/scripts/Training.pl hts/scripts/Config.pm (It can take several hours...)*/     
-      cmdLine = db.getExternal(db.PERLPATH) + "/perl " + voicedir +"hts/scripts/Training.pl " + voicedir + "hts/scripts/Config.pm";      
-      launchProcWithLogFile(cmdLine, "", voicedir);
-        
-      return true;
-    }
-    
-    
-    /**
-     * A general process launcher for the various tasks
-     * (copied from ESTCaller.java)
-     * @param cmdLine the command line to be launched.
-     * @param task a task tag for error messages, such as "Pitchmarks" or "LPC".
-     * @param the basename of the file currently processed, for error messages.
-     */
-    private void launchProcWithLogFile( String cmdLine, String task, String voicedir ) {
-        
-        Process proc = null;
-        BufferedReader procStdout = null;
-        String line = null;
-        
-        Date today;
-        String output;
-        SimpleDateFormat formatter;
-        formatter = new SimpleDateFormat("yyyy.MM.dd-H:mm:ss", new Locale("en","US"));
-        today = new Date();
-        output = formatter.format(today);           
-        String logFile = voicedir+"hts/log-"+output;
-        
-        System.out.println("\nRunning: "+ cmdLine);
-        System.out.println("\nThe training procedure can take several hours...");
-        System.out.println("Detailed information about the training status can be found in the logfile:\n  " + logFile);
-        System.out.println("\nTraining voice: " + db.getProp(db.VOICENAME));
-        System.out.println("The following is general information about execution of training steps:");
-        // String[] cmd = null; // Java 5.0 compliant code
-        
-        try {
-            FileWriter log = new FileWriter(logFile);
-            /* Java 5.0 compliant code below. */
-            /* Hook the command line to the process builder: */
-            ProcessBuilder pb = new  ProcessBuilder(cmdLine.split(" "));
-            pb.directory(new File( db.getProp(db.ROOTDIR) ));
-            pb.redirectErrorStream(true);
-            /* Launch the process: */
-            proc = pb.start();
-            
-            /* Collect process's combined stdout & stderr send it to System.out: */
-            procStdout = new BufferedReader(
-            new InputStreamReader(proc.getInputStream()));
-            while ((line = procStdout.readLine()) != null) {
-            	if(line.contains("Start "))
-                    System.out.println( "\nStep: " + line );
-                  log.write(line+"\n");
-            }
-            
-            /* Wait and check the exit value */
-            proc.waitFor();
-            if ( proc.exitValue() != 0 ) {
-                BufferedReader errReader = new BufferedReader(new InputStreamReader(proc.getErrorStream()));
-                while ((line = errReader.readLine()) != null) {
-                    System.err.println("ERR> "+line);
-                }
-                errReader.close();
-                throw new RuntimeException( task + " computation failed on file [" + voicedir + "]!\n"
-                        + "Command line was: [" + cmdLine + "]." );
-            }
-            log.close();
-        }
-        catch ( IOException e ) {
-            throw new RuntimeException( task + " computation provoked an IOException on file [" + voicedir + "].", e );
-        }
-        catch ( InterruptedException e ) {
-            throw new RuntimeException( task + " computation interrupted on file [" + voicedir + "].", e );
-        }
-        
-    }    
-    
-    
-    /**
-     * Provide the progress of computation, in percent, or -1 if
-     * that feature is not implemented.
-     * @return -1 if not implemented, or an integer between 0 and 100.
-     */
-    public int getProgress(){
-        return -1;
-    }
-    
 }

marytts-builder/src/main/java/marytts/tools/voiceimport/HalfPhoneLabelFeatureAligner.java

@@ -19,34 +19,29 @@
  */
 package marytts.tools.voiceimport;
 
-
-
 /**
  * @author marc
  *
  */
 public class HalfPhoneLabelFeatureAligner extends PhoneLabelFeatureAligner {
-    
-    public String getName(){
-        return "HalfPhoneLabelFeatureAligner";
-    }
-    
-    public HalfPhoneLabelFeatureAligner(){
-    }
-    
-    @Override
-    protected void customInitialisation()
-    {
-        featureComputer = (HalfPhoneUnitFeatureComputer) db.getComponent("HalfPhoneUnitFeatureComputer");
-        allophoneExtractor = (AllophonesExtractor) db.getComponent("AllophonesExtractor");
-        labelComputer =  (HalfPhoneUnitLabelComputer) db.getComponent("HalfPhoneUnitLabelComputer");
-        transcriptionAligner = (TranscriptionAligner) db.getComponent("TranscriptionAligner");
-        featsExt = ".hpfeats";
-        labExt = ".hplab";
-        featsDir = db.getProp(db.HALFPHONEFEATUREDIR);
-        labDir = db.getProp(db.HALFPHONELABDIR);
-    }
 
+	public String getName() {
+		return "HalfPhoneLabelFeatureAligner";
+	}
+
+	public HalfPhoneLabelFeatureAligner() {
+	}
+
+	@Override
+	protected void customInitialisation() {
+		featureComputer = (HalfPhoneUnitFeatureComputer) db.getComponent("HalfPhoneUnitFeatureComputer");
+		allophoneExtractor = (AllophonesExtractor) db.getComponent("AllophonesExtractor");
+		labelComputer = (HalfPhoneUnitLabelComputer) db.getComponent("HalfPhoneUnitLabelComputer");
+		transcriptionAligner = (TranscriptionAligner) db.getComponent("TranscriptionAligner");
+		featsExt = ".hpfeats";
+		labExt = ".hplab";
+		featsDir = db.getProp(db.HALFPHONEFEATUREDIR);
+		labDir = db.getProp(db.HALFPHONELABDIR);
+	}
 
 }
-

marytts-builder/src/main/java/marytts/tools/voiceimport/HalfPhoneUnitFeatureComputer.java

@@ -30,105 +30,88 @@ import marytts.util.io.FileUtils;
  * @author marc
  *
  */
-public class HalfPhoneUnitFeatureComputer extends PhoneUnitFeatureComputer 
-{
-    public static final String[] HALFPHONE_FEATURES = new String[] {
-        "halfphone_lr",
-        "halfphone_unitname"
-    };
-    public static final String HALFPHONE_UNITNAME = "halfphone_unitname";
-    
-    public String getName(){
-        return "HalfPhoneUnitFeatureComputer";
-    }
-    
-    public HalfPhoneUnitFeatureComputer(){        
-        featsExt = ".hpfeats";
-        FEATUREDIR = "HalfPhoneUnitFeatureComputer.featureDir";
-        ALLOPHONES = "HalfPhoneUnitFeatureComputer.allophonesDir";
-        FEATURELIST = "HalfPhoneUnitFeatureComputer.featureFile";
-        MARYSERVERHOST = "HalfPhoneUnitFeatureComputer.maryServerHost";
-        MARYSERVERPORT = "HalfPhoneUnitFeatureComputer.maryServerPort";   
-    }
-    
-    @Override
-    protected void initialiseComp()
-    throws Exception
-    {       
-        locale = db.getProp(db.LOCALE);        
-        mary = null; // initialised only if needed   
-        unitfeatureDir = new File(getProp(FEATUREDIR));
-        if (!unitfeatureDir.exists()){
-            System.out.print(FEATUREDIR+" "+getProp(FEATUREDIR)
-                    +" does not exist; ");
-            if (!unitfeatureDir.mkdir()){
-                throw new Error("Could not create FEATUREDIR");
-            }
-            System.out.print("Created successfully.\n");
-        }
-        maryInputType  = "ALLOPHONES";
-        maryOutputType = "HALFPHONE_TARGETFEATURES";
-    }
-    
-      public SortedMap<String,String> getDefaultProps(DatabaseLayout theDb){
-        this.db = theDb;
-       if (props == null){
-           props = new TreeMap<String, String>();
-           props.put(FEATUREDIR, db.getProp(db.ROOTDIR)
-                        +"halfphonefeatures"
-                        +System.getProperty("file.separator"));
-           props.put(ALLOPHONES, db.getProp(db.ROOTDIR)
-                   +"allophones"
-                   +System.getProperty("file.separator"));
-           props.put(FEATURELIST,
-                   db.getProp(db.CONFIGDIR) + "features.txt");
-           props.put(MARYSERVERHOST,"localhost");
-           props.put(MARYSERVERPORT,"59125");
-       } 
-       return props;
-      }
-      
-      protected void setupHelp(){
-         props2Help = new TreeMap<String, String>();
-         props2Help.put(FEATUREDIR, "directory containing the halfphone features." 
-                 +"Will be created if it does not exist.");
-         props2Help.put(ALLOPHONES, "Directory of corrected Allophones files.");
-         props2Help.put(MARYSERVERHOST,"the host were the Mary server is running, default: \"localhost\"");
-         props2Help.put(MARYSERVERPORT,"the port were the Mary server is listening, default: \"59125\"");
-     }
-      
-      
-      @Override
-      protected void loadFeatureList()
-      throws IOException
-      {
-          File featureFile = new File(getProp(FEATURELIST));
-          if (!featureFile.exists()) {
-              System.out.println("No feature file: '"+getProp(FEATURELIST)+"'");
-          } else {
-              System.out.println("Loading features from file "+getProp(FEATURELIST));
-              try {
-                  featureList = FileUtils.getFileAsString(featureFile, "UTF-8");
-                  featureList = featureList.replaceAll("\\s+", " ");
-                  // Make sure specific halfphone features are included:
-                  for (String f : HALFPHONE_FEATURES) {
-                      if (!featureList.contains(f)) {
-                          featureList = f + " " + featureList;
-                      }
-                  }
-                  if (!featureList.startsWith(HALFPHONE_UNITNAME)) {
-                      // HALFPHONE_UNITNAME must be the first one in the list
-                      featureList = featureList.replaceFirst("\\s+"+HALFPHONE_UNITNAME+"\\s+", " ");
-                      featureList = HALFPHONE_UNITNAME + " " + featureList;
+public class HalfPhoneUnitFeatureComputer extends PhoneUnitFeatureComputer {
+	public static final String[] HALFPHONE_FEATURES = new String[] { "halfphone_lr", "halfphone_unitname" };
+	public static final String HALFPHONE_UNITNAME = "halfphone_unitname";
 
-                  }
-              } catch (IOException e) {
-                  IOException ioe = new IOException("Cannot read feature list");
-                  ioe.initCause(e);
-                  throw ioe;
-              }
-          }
+	public String getName() {
+		return "HalfPhoneUnitFeatureComputer";
+	}
 
-      }
+	public HalfPhoneUnitFeatureComputer() {
+		featsExt = ".hpfeats";
+		FEATUREDIR = "HalfPhoneUnitFeatureComputer.featureDir";
+		ALLOPHONES = "HalfPhoneUnitFeatureComputer.allophonesDir";
+		FEATURELIST = "HalfPhoneUnitFeatureComputer.featureFile";
+		MARYSERVERHOST = "HalfPhoneUnitFeatureComputer.maryServerHost";
+		MARYSERVERPORT = "HalfPhoneUnitFeatureComputer.maryServerPort";
+	}
+
+	@Override
+	protected void initialiseComp() throws Exception {
+		locale = db.getProp(db.LOCALE);
+		mary = null; // initialised only if needed
+		unitfeatureDir = new File(getProp(FEATUREDIR));
+		if (!unitfeatureDir.exists()) {
+			System.out.print(FEATUREDIR + " " + getProp(FEATUREDIR) + " does not exist; ");
+			if (!unitfeatureDir.mkdir()) {
+				throw new Error("Could not create FEATUREDIR");
+			}
+			System.out.print("Created successfully.\n");
+		}
+		maryInputType = "ALLOPHONES";
+		maryOutputType = "HALFPHONE_TARGETFEATURES";
+	}
+
+	public SortedMap<String, String> getDefaultProps(DatabaseLayout theDb) {
+		this.db = theDb;
+		if (props == null) {
+			props = new TreeMap<String, String>();
+			props.put(FEATUREDIR, db.getProp(db.ROOTDIR) + "halfphonefeatures" + System.getProperty("file.separator"));
+			props.put(ALLOPHONES, db.getProp(db.ROOTDIR) + "allophones" + System.getProperty("file.separator"));
+			props.put(FEATURELIST, db.getProp(db.CONFIGDIR) + "features.txt");
+			props.put(MARYSERVERHOST, "localhost");
+			props.put(MARYSERVERPORT, "59125");
+		}
+		return props;
+	}
+
+	protected void setupHelp() {
+		props2Help = new TreeMap<String, String>();
+		props2Help.put(FEATUREDIR, "directory containing the halfphone features." + "Will be created if it does not exist.");
+		props2Help.put(ALLOPHONES, "Directory of corrected Allophones files.");
+		props2Help.put(MARYSERVERHOST, "the host were the Mary server is running, default: \"localhost\"");
+		props2Help.put(MARYSERVERPORT, "the port were the Mary server is listening, default: \"59125\"");
+	}
+
+	@Override
+	protected void loadFeatureList() throws IOException {
+		File featureFile = new File(getProp(FEATURELIST));
+		if (!featureFile.exists()) {
+			System.out.println("No feature file: '" + getProp(FEATURELIST) + "'");
+		} else {
+			System.out.println("Loading features from file " + getProp(FEATURELIST));
+			try {
+				featureList = FileUtils.getFileAsString(featureFile, "UTF-8");
+				featureList = featureList.replaceAll("\\s+", " ");
+				// Make sure specific halfphone features are included:
+				for (String f : HALFPHONE_FEATURES) {
+					if (!featureList.contains(f)) {
+						featureList = f + " " + featureList;
+					}
+				}
+				if (!featureList.startsWith(HALFPHONE_UNITNAME)) {
+					// HALFPHONE_UNITNAME must be the first one in the list
+					featureList = featureList.replaceFirst("\\s+" + HALFPHONE_UNITNAME + "\\s+", " ");
+					featureList = HALFPHONE_UNITNAME + " " + featureList;
+
+				}
+			} catch (IOException e) {
+				IOException ioe = new IOException("Cannot read feature list");
+				ioe.initCause(e);
+				throw ioe;
+			}
+		}
+
+	}
 }
-

marytts-builder/src/main/java/marytts/tools/voiceimport/HalfPhoneUnitLabelComputer.java

@@ -19,7 +19,6 @@
  */
 package marytts.tools.voiceimport;
 
-
 import java.io.File;
 import java.io.IOException;
 import java.util.ArrayList;
@@ -37,184 +36,187 @@ import org.apache.commons.lang.ArrayUtils;
 
 /**
  * Compute unit labels from phone labels.
+ * 
  * @author schroed
  *
  */
-public class HalfPhoneUnitLabelComputer extends PhoneUnitLabelComputer
-{    
+public class HalfPhoneUnitLabelComputer extends PhoneUnitLabelComputer {
 
-    private String ENERGYBASEDTRANSIENTSPLITTING = getName() + ".energyBasedTransientSplitting";
-    private boolean energyBasedTransientSplitting;
-    private String energyExt = ".energy";
-    // these could be user configurable properties, but at this stage, it's too easy to screw up:
-    private double windowSizeInSeconds = 0.005;
-    private double skipSizeInSeconds = 0.0025;
+	private String ENERGYBASEDTRANSIENTSPLITTING = getName() + ".energyBasedTransientSplitting";
+	private boolean energyBasedTransientSplitting;
+	private String energyExt = ".energy";
+	// these could be user configurable properties, but at this stage, it's too easy to screw up:
+	private double windowSizeInSeconds = 0.005;
+	private double skipSizeInSeconds = 0.0025;
 
-    public String getName(){
-        return "HalfPhoneUnitLabelComputer";      
-    }
-    
-    public HalfPhoneUnitLabelComputer(){
-    }
+	public String getName() {
+		return "HalfPhoneUnitLabelComputer";
+	}
 
-    @Override
-    protected void initialiseComp() throws Exception {
-    	super.initialiseComp();
-    	unitlabelDir = new File(db.getProp(DatabaseLayout.HALFPHONELABDIR));
-        unitlabelExt = db.getProp(DatabaseLayout.HALFPHONELABEXT);
-    }
-    
-    public SortedMap<String, String> getDefaultProps(DatabaseLayout db) {
-    	this.db = db;
-    	if (props == null){
-    		props = new TreeMap<String, String>();
-    		props.put(ENERGYBASEDTRANSIENTSPLITTING, "false");
-    	}
-    	return props;
-    }
-    
-    protected void setupHelp(){
-        props2Help = new TreeMap<String, String>();
-        props2Help.put(ENERGYBASEDTRANSIENTSPLITTING,
-                "Whether to analyze energy in the speech signal to determine midpoints of transient phones (plosives).");
-    } 
-    
-    @Override
-    public boolean compute() throws Exception {
-        energyBasedTransientSplitting = Boolean.parseBoolean(db.getProperty(ENERGYBASEDTRANSIENTSPLITTING));
-        return super.compute();
-    }
-    
-    @Override
-    protected List<Double> getMidTimes(List<String> labels, List<Double> endTimes) {
-        assert labels.size() == endTimes.size();
+	public HalfPhoneUnitLabelComputer() {
+	}
 
-        List<Double> midTimes = new ArrayList<Double>(endTimes.size());
-        double startTime = 0;
-        for (int i = 0; i < labels.size(); i++) {
-            String label = labels.get(i);
-            double endTime = endTimes.get(i);
+	@Override
+	protected void initialiseComp() throws Exception {
+		super.initialiseComp();
+		unitlabelDir = new File(db.getProp(DatabaseLayout.HALFPHONELABDIR));
+		unitlabelExt = db.getProp(DatabaseLayout.HALFPHONELABEXT);
+	}
 
-            boolean isTransient = false;
-            double peakTime = Double.NaN;
-            if (energyBasedTransientSplitting) {
-                try {
-                    Allophone allophone = db.getAllophoneSet().getAllophone(label);
-                    isTransient = allophone.isPlosive() || allophone.isAffricate();
-                    if (isTransient) {
-                        peakTime = getEnergyPeak(startTime, endTime);
-                    }
-                } catch (NullPointerException e) {
-                    // ignore for now
-                } catch (IOException e) {
-                    // ignore for now
-                }
-            }
+	public SortedMap<String, String> getDefaultProps(DatabaseLayout db) {
+		this.db = db;
+		if (props == null) {
+			props = new TreeMap<String, String>();
+			props.put(ENERGYBASEDTRANSIENTSPLITTING, "false");
+		}
+		return props;
+	}
+
+	protected void setupHelp() {
+		props2Help = new TreeMap<String, String>();
+		props2Help.put(ENERGYBASEDTRANSIENTSPLITTING,
+				"Whether to analyze energy in the speech signal to determine midpoints of transient phones (plosives).");
+	}
+
+	@Override
+	public boolean compute() throws Exception {
+		energyBasedTransientSplitting = Boolean.parseBoolean(db.getProperty(ENERGYBASEDTRANSIENTSPLITTING));
+		return super.compute();
+	}
+
+	@Override
+	protected List<Double> getMidTimes(List<String> labels, List<Double> endTimes) {
+		assert labels.size() == endTimes.size();
+
+		List<Double> midTimes = new ArrayList<Double>(endTimes.size());
+		double startTime = 0;
+		for (int i = 0; i < labels.size(); i++) {
+			String label = labels.get(i);
+			double endTime = endTimes.get(i);
+
+			boolean isTransient = false;
+			double peakTime = Double.NaN;
+			if (energyBasedTransientSplitting) {
+				try {
+					Allophone allophone = db.getAllophoneSet().getAllophone(label);
+					isTransient = allophone.isPlosive() || allophone.isAffricate();
+					if (isTransient) {
+						peakTime = getEnergyPeak(startTime, endTime);
+					}
+				} catch (NullPointerException e) {
+					// ignore for now
+				} catch (IOException e) {
+					// ignore for now
+				}
+			}
+
+			double midTime;
+			if (isTransient && !Double.isNaN(peakTime)) {
+				midTime = peakTime;
+			} else {
+				midTime = (startTime + endTime) / 2;
+			}
+			midTimes.add(midTime);
+			startTime = endTime;
+		}
+		return midTimes;
+	}
+
+	/**
+	 * Get time of energy peak difference between startTime and endTime, based on energy analysis of the wav file for the current
+	 * baseName.
+	 * <p>
+	 * The energy analysis (based on the provided parameters {@link #windowSizeInSeconds} and {@link #skipSizeInSeconds}) is saved
+	 * to a binary file, which is reused if present (and if the parameter values match those encountered in the file header).
+	 * 
+	 * @param startTime
+	 *            of energy analysis
+	 * @param endTime
+	 *            of energy analysis
+	 * @return the time of the greatest increase in energy between startTime and endTime, or {@link Double#NaN} if no such time
+	 *         can be determined from the signal (this is then handled in {@link #getMidTimes(List, List)})
+	 * @throws IOException
+	 *             if the energy analysis file cannot be read or (initially) created
+	 * @see EnergyContourRms#WriteEnergyFile(EnergyContourRms, String)
+	 */
+	private double getEnergyPeak(double startTime, double endTime) throws IOException {
+		// determine wav file name and energy analysis file name:
+		String wavDir = db.getProperty(DatabaseLayout.WAVDIR);
+		String baseName = bnl.getName(basenameIndex);
+		String wavExt = db.getProperty(DatabaseLayout.WAVEXT);
+		File wavFile = new File(wavDir, baseName + wavExt);
+		File energyFile = new File(unitlabelDir, baseName + energyExt);
+
+		// load or create energy analysis file:
+		EnergyContourRms energyContourRMS;
+		try {
+			energyContourRMS = EnergyContourRms.ReadEnergyFile(energyFile.getAbsolutePath());
+			if (energyContourRMS.header.windowSizeInSeconds != windowSizeInSeconds
+					|| energyContourRMS.header.skipSizeInSeconds != skipSizeInSeconds) {
+				logger.debug("File header of " + energyFile.getAbsolutePath()
+						+ " has unexpected parameter values! Will re-analyze...");
+				throw new IOException();
+			}
+		} catch (IOException e) {
+			logger.info("Analyzing " + wavFile.getAbsolutePath() + " and saving result to " + energyFile.getAbsolutePath());
+			energyContourRMS = new EnergyContourRms(wavFile.getAbsolutePath(), energyFile.getAbsolutePath(), windowSizeInSeconds,
+					skipSizeInSeconds);
+		}
+
+		// get energy analysis frames between startTime and endTime from energy contour:
+		double[] energyContour = energyContourRMS.contour;
+		int startFrame = SignalProcUtils.time2frameIndex(startTime, windowSizeInSeconds, skipSizeInSeconds);
+		int endFrame = SignalProcUtils.time2frameIndex(endTime, windowSizeInSeconds, skipSizeInSeconds);
+		double[] energyLocalContour = ArrayUtils.subarray(energyContour, startFrame, endFrame);
+
+		// get framewise differences:
+		double[] energyDiffs = MathUtils.diff(energyLocalContour);
+		// we need more than one diff frame:
+		if (energyDiffs.length < 2) {
+			return Double.NaN;
+		}
+		// find frame index of peak diff:
+		int peakLocalFrame = MathUtils.findGlobalPeakLocation(energyDiffs);
+		int peakGlobalFrame = startFrame + peakLocalFrame;
+		// convert frame index to time, adding half a window because diffs are between frames:
+		double peakTime = SignalProcUtils.frameIndex2Time(peakGlobalFrame, windowSizeInSeconds, skipSizeInSeconds)
+				+ windowSizeInSeconds / 2;
+
+		// adjust peak diff time to lie inside time range:
+		if (peakTime < startTime) {
+			peakTime = startTime;
+		} else if (peakTime > endTime) {
+			peakTime = endTime;
+		}
+		return peakTime;
+	}
+
+	@Override
+	@Deprecated
+	protected String[] toUnitLabels(String[] phoneLabels) {
+		// We will create exactly two half phones for every phone:
+		String[] halfPhoneLabels = new String[2 * phoneLabels.length];
+		float startTime = 0;
+		int unitIndex = 0;
+		for (int i = 0; i < phoneLabels.length; i++) {
+			unitIndex++;
+			StringTokenizer st = new StringTokenizer(phoneLabels[i]);
+			String endTimeString = st.nextToken();
+			String dummyNumber = st.nextToken();
+			String phone = st.nextToken();
+			assert !st.hasMoreTokens();
+			float endTime = Float.parseFloat(endTimeString);
+			float duration = endTime - startTime;
+			assert duration > 0 : "Duration is not > 0 for phone " + i + " (" + phone + ")";
+			float midTime = startTime + duration / 2;
+			String leftUnitLine = midTime + " " + unitIndex + " " + phone + "_L";
+			unitIndex++;
+			String rightUnitLine = endTime + " " + unitIndex + " " + phone + "_R";
+			halfPhoneLabels[2 * i] = leftUnitLine;
+			halfPhoneLabels[2 * i + 1] = rightUnitLine;
+			startTime = endTime;
+		}
+		return halfPhoneLabels;
+	}
 
-            double midTime;
-            if (isTransient && !Double.isNaN(peakTime)) {
-                midTime = peakTime;
-            } else {
-                midTime = (startTime + endTime) / 2;
-            }
-            midTimes.add(midTime);
-            startTime = endTime;
-        }
-        return midTimes;
-    }
-    
-    /**
-     * Get time of energy peak difference between startTime and endTime, based on energy analysis of the wav file for the current baseName.
-     * <p>
-     * The energy analysis (based on the provided parameters {@link #windowSizeInSeconds} and {@link #skipSizeInSeconds}) is saved
-     * to a binary file, which is reused if present (and if the parameter values match those encountered in the file header).
-     * 
-     * @param startTime
-     *            of energy analysis
-     * @param endTime
-     *            of energy analysis
-     * @return the time of the greatest increase in energy between startTime and endTime, or {@link Double#NaN} if no such
-     *         time can be determined from the signal (this is then handled in {@link #getMidTimes(List, List)})
-     * @throws IOException
-     *             if the energy analysis file cannot be read or (initially) created
-     * @see EnergyContourRms#WriteEnergyFile(EnergyContourRms, String)
-     */
-    private double getEnergyPeak(double startTime, double endTime) throws IOException {
-        // determine wav file name and energy analysis file name:
-        String wavDir = db.getProperty(DatabaseLayout.WAVDIR);
-        String baseName = bnl.getName(basenameIndex);
-        String wavExt = db.getProperty(DatabaseLayout.WAVEXT);
-        File wavFile = new File(wavDir, baseName + wavExt);
-        File energyFile = new File(unitlabelDir, baseName + energyExt);
-        
-        // load or create energy analysis file:
-        EnergyContourRms energyContourRMS;
-        try {
-            energyContourRMS = EnergyContourRms.ReadEnergyFile(energyFile.getAbsolutePath());
-            if (energyContourRMS.header.windowSizeInSeconds != windowSizeInSeconds || energyContourRMS.header.skipSizeInSeconds != skipSizeInSeconds) {
-                logger.debug("File header of " + energyFile.getAbsolutePath() + " has unexpected parameter values! Will re-analyze...");
-                throw new IOException();
-            }
-        } catch (IOException e) {
-            logger.info("Analyzing " + wavFile.getAbsolutePath() + " and saving result to " + energyFile.getAbsolutePath());
-            energyContourRMS = new EnergyContourRms(wavFile.getAbsolutePath(), energyFile.getAbsolutePath(), windowSizeInSeconds, skipSizeInSeconds);
-        }
-        
-        // get energy analysis frames between startTime and endTime from energy contour:
-        double[] energyContour = energyContourRMS.contour;
-        int startFrame = SignalProcUtils.time2frameIndex(startTime, windowSizeInSeconds, skipSizeInSeconds);
-        int endFrame = SignalProcUtils.time2frameIndex(endTime, windowSizeInSeconds, skipSizeInSeconds);
-        double[] energyLocalContour = ArrayUtils.subarray(energyContour, startFrame, endFrame);
-        
-        // get framewise differences:
-        double[] energyDiffs = MathUtils.diff(energyLocalContour);
-        // we need more than one diff frame:
-        if (energyDiffs.length < 2) {
-            return Double.NaN;
-        }
-        // find frame index of peak diff:
-        int peakLocalFrame = MathUtils.findGlobalPeakLocation(energyDiffs);
-        int peakGlobalFrame = startFrame + peakLocalFrame;
-        // convert frame index to time, adding half a window because diffs are between frames:
-        double peakTime = SignalProcUtils.frameIndex2Time(peakGlobalFrame, windowSizeInSeconds, skipSizeInSeconds) + windowSizeInSeconds / 2;
-        
-        // adjust peak diff time to lie inside time range:
-        if (peakTime < startTime) {
-            peakTime = startTime;
-        } else if (peakTime > endTime) {
-            peakTime = endTime;
-        }
-        return peakTime;
-    }
-     
-    @Override
-    @Deprecated
-    protected String[] toUnitLabels(String[] phoneLabels)
-    {
-        // We will create exactly two half phones for every phone:
-        String[] halfPhoneLabels = new String[2*phoneLabels.length];
-        float startTime = 0;
-        int unitIndex = 0;
-        for (int i=0; i<phoneLabels.length; i++) {
-            unitIndex++;
-            StringTokenizer st = new StringTokenizer(phoneLabels[i]);
-            String endTimeString = st.nextToken();
-            String dummyNumber = st.nextToken();
-            String phone = st.nextToken();
-            assert !st.hasMoreTokens();
-            float endTime = Float.parseFloat(endTimeString);
-            float duration = endTime - startTime;
-            assert duration > 0 : "Duration is not > 0 for phone "+i+" ("+phone+")";
-            float midTime = startTime + duration/2;
-            String leftUnitLine = midTime + " " + unitIndex + " " + phone + "_L";
-            unitIndex++;
-            String rightUnitLine = endTime + " " + unitIndex + " " + phone + "_R";
-            halfPhoneLabels[2*i] = leftUnitLine;
-            halfPhoneLabels[2*i+1] = rightUnitLine;
-            startTime = endTime;
-        }
-        return halfPhoneLabels;
-    }
-    
 }
-

marytts-builder/src/main/java/marytts/tools/voiceimport/HalfPhoneUnitfileWriter.java

@@ -27,57 +27,50 @@ import java.util.TreeMap;
  * @author marc
  *
  */
-public class HalfPhoneUnitfileWriter extends PhoneUnitfileWriter
-{
+public class HalfPhoneUnitfileWriter extends PhoneUnitfileWriter {
 
-   
-    public String getName(){
-        return "HalfPhoneUnitfileWriter";
-    }
-    
-    public HalfPhoneUnitfileWriter(){
-        unitlabelExt = ".hplab";
-        UNITFILE = "HalfPhoneUnitfileWriter.unitFile";
-    }
-    
-    @Override
-    protected void initialiseComp()
-    throws Exception
-    {
-        maryDir = new File(db.getProp(DatabaseLayout.FILEDIR));
-        
-        samplingRate = Integer.parseInt(db.getProp(DatabaseLayout.SAMPLINGRATE));
-        pauseSymbol = db.getAllophoneSet().getSilence().name();
-    
-        unitFileName = getProp(UNITFILE);
-        unitlabelDir = new File(db.getProp(DatabaseLayout.HALFPHONELABDIR));
-        if (!unitlabelDir.exists()){
-            System.out.print(DatabaseLayout.HALFPHONELABDIR+" "+db.getProp(DatabaseLayout.HALFPHONELABDIR)
-                    +" does not exist; ");
-            if (!unitlabelDir.mkdir()){
-                throw new Exception("Could not create HALFPHONELABDIR");
-            }
-            System.out.print("Created successfully.\n");
-        } 
-        aligner = new HalfPhoneLabelFeatureAligner();
-        db.initialiseComponent(aligner);        
-    }
-    
-    public SortedMap<String, String> getDefaultProps(DatabaseLayout db) {
-        this.db = db;
-        if (props == null){
-            props = new TreeMap<String, String>();
-            props.put(UNITFILE, db.getProp(DatabaseLayout.FILEDIR)
-                    +"halfphoneUnits"+db.getProp(DatabaseLayout.MARYEXT));           
-        }
-        return props;
-    }
-    
-    protected void setupHelp(){
-        props2Help = new TreeMap<String, String>();
-        props2Help.put(UNITFILE, "file containing all halfphone units. Will be created by this module");           
-    }
-    
+	public String getName() {
+		return "HalfPhoneUnitfileWriter";
+	}
+
+	public HalfPhoneUnitfileWriter() {
+		unitlabelExt = ".hplab";
+		UNITFILE = "HalfPhoneUnitfileWriter.unitFile";
+	}
+
+	@Override
+	protected void initialiseComp() throws Exception {
+		maryDir = new File(db.getProp(DatabaseLayout.FILEDIR));
+
+		samplingRate = Integer.parseInt(db.getProp(DatabaseLayout.SAMPLINGRATE));
+		pauseSymbol = db.getAllophoneSet().getSilence().name();
+
+		unitFileName = getProp(UNITFILE);
+		unitlabelDir = new File(db.getProp(DatabaseLayout.HALFPHONELABDIR));
+		if (!unitlabelDir.exists()) {
+			System.out.print(DatabaseLayout.HALFPHONELABDIR + " " + db.getProp(DatabaseLayout.HALFPHONELABDIR)
+					+ " does not exist; ");
+			if (!unitlabelDir.mkdir()) {
+				throw new Exception("Could not create HALFPHONELABDIR");
+			}
+			System.out.print("Created successfully.\n");
+		}
+		aligner = new HalfPhoneLabelFeatureAligner();
+		db.initialiseComponent(aligner);
+	}
+
+	public SortedMap<String, String> getDefaultProps(DatabaseLayout db) {
+		this.db = db;
+		if (props == null) {
+			props = new TreeMap<String, String>();
+			props.put(UNITFILE, db.getProp(DatabaseLayout.FILEDIR) + "halfphoneUnits" + db.getProp(DatabaseLayout.MARYEXT));
+		}
+		return props;
+	}
+
+	protected void setupHelp() {
+		props2Help = new TreeMap<String, String>();
+		props2Help.put(UNITFILE, "file containing all halfphone units. Will be created by this module");
+	}
 
 }
-

marytts-builder/src/main/java/marytts/tools/voiceimport/HnmTimelineMaker.java

@@ -52,396 +52,435 @@ import marytts.util.io.FileUtils;
 import marytts.util.math.MathUtils;
 import marytts.util.signal.SignalProcUtils;
 
-
 /**
- * HnmTimelineMaker class takes a database root directory and a list of basenames,
- * and converts the related wav files into a hnm timeline in Mary format.
+ * HnmTimelineMaker class takes a database root directory and a list of basenames, and converts the related wav files into a hnm
+ * timeline in Mary format.
  * 
  * @author Oytun T&uumlrk
  */
-public class HnmTimelineMaker extends VoiceImportComponent
-{ 
-    protected DatabaseLayout db = null;
-    protected int percent = 0;
-    
-    protected String hnmAnalysisFileExt = ".ana";
-    public final String HNMTIMELINE = "HnmTimelineMaker.hnmTimeline";
-    public final String HNMANADIR = "HnmTimelineMaker.hnmAnalysisDir";
-    
-    public final String PMDIR = "db.pmDir";
-    public final String PMEXT = "db.pmExtension";
+public class HnmTimelineMaker extends VoiceImportComponent {
+	protected DatabaseLayout db = null;
+	protected int percent = 0;
 
-    public String getName(){
-        return "HnmTimelineMaker";
-    }
-    
-    public SortedMap<String,String> getDefaultProps(DatabaseLayout theDb){
-        this.db = theDb;
-        if (props == null)
-        {
-            HntmAnalyzerParams analysisParams = new HntmAnalyzerParams();
-            HntmSynthesizerParams synthesisParams = new HntmSynthesizerParams();
-            
-            props = new TreeMap<String, String>();
+	protected String hnmAnalysisFileExt = ".ana";
+	public final String HNMTIMELINE = "HnmTimelineMaker.hnmTimeline";
+	public final String HNMANADIR = "HnmTimelineMaker.hnmAnalysisDir";
 
-            props.put(HNMTIMELINE, db.getProp(db.FILEDIR)
-                    +"timeline_hnm"+db.getProp(db.MARYEXT));
-    
-            props.put(HNMANADIR, db.getProp(db.ROOTDIR)
-                    +"hna"
-                    +System.getProperty("file.separator"));
-            
-            props.put("HnmTimelineMaker.noiseModel", String.valueOf(analysisParams.noiseModel));
-            props.put("HnmTimelineMaker.numFiltStages", String.valueOf(analysisParams.hnmPitchVoicingAnalyzerParams.numFilteringStages));
-            props.put("HnmTimelineMaker.medianFiltLen", String.valueOf(analysisParams.hnmPitchVoicingAnalyzerParams.medianFilterLength));
-            props.put("HnmTimelineMaker.maFiltLen", String.valueOf(analysisParams.hnmPitchVoicingAnalyzerParams.movingAverageFilterLength));
-            props.put("HnmTimelineMaker.cumAmpTh", String.valueOf(analysisParams.hnmPitchVoicingAnalyzerParams.cumulativeAmpThreshold));
-            props.put("HnmTimelineMaker.maxAmpTh", String.valueOf(analysisParams.hnmPitchVoicingAnalyzerParams.maximumAmpThresholdInDB));
-            props.put("HnmTimelineMaker.harmDevPercent", String.valueOf(analysisParams.hnmPitchVoicingAnalyzerParams.harmonicDeviationPercent));
-            props.put("HnmTimelineMaker.sharpPeakAmpDiff", String.valueOf(analysisParams.hnmPitchVoicingAnalyzerParams.sharpPeakAmpDiffInDB));
-            props.put("HnmTimelineMaker.minHarmonics", String.valueOf(analysisParams.hnmPitchVoicingAnalyzerParams.minimumTotalHarmonics));
-            props.put("HnmTimelineMaker.maxHarmonics", String.valueOf(analysisParams.hnmPitchVoicingAnalyzerParams.maximumTotalHarmonics));
-            props.put("HnmTimelineMaker.minVoicedFreq", String.valueOf(analysisParams.hnmPitchVoicingAnalyzerParams.minimumVoicedFrequencyOfVoicing));
-            props.put("HnmTimelineMaker.maxVoicedFreq", String.valueOf(analysisParams.hnmPitchVoicingAnalyzerParams.maximumVoicedFrequencyOfVoicing));
-            props.put("HnmTimelineMaker.maxFreqVoicingFinalShift", String.valueOf(analysisParams.hnmPitchVoicingAnalyzerParams.maximumFrequencyOfVoicingFinalShift));
-            props.put("HnmTimelineMaker.neighsPercent", String.valueOf(analysisParams.hnmPitchVoicingAnalyzerParams.neighsPercent));
-            props.put("HnmTimelineMaker.harmCepsOrder", String.valueOf(analysisParams.harmonicPartCepstrumOrder));
-            props.put("HnmTimelineMaker.regCepWarpMethod", String.valueOf(analysisParams.regularizedCepstrumWarpingMethod));
-            props.put("HnmTimelineMaker.regCepsLambda", String.valueOf(analysisParams.regularizedCepstrumLambdaHarmonic));
-            props.put("HnmTimelineMaker.noiseLpOrder", String.valueOf(analysisParams.noisePartLpOrder));
-            props.put("HnmTimelineMaker.preCoefNoise", String.valueOf(analysisParams.preemphasisCoefNoise));
-            props.put("HnmTimelineMaker.hpfBeforeNoiseAnalysis", String.valueOf(analysisParams.hpfBeforeNoiseAnalysis));
-            props.put("HnmTimelineMaker.harmNumPer", String.valueOf(analysisParams.numPeriodsHarmonicsExtraction));
-        }
-        
-        return props;
-    }
-    
-    protected void setupHelp(){         
-        props2Help = new TreeMap<String, String>();
-        
-        props2Help.put(HNMTIMELINE,"file containing all hnm noise waveform files. Will be created by this module");  
-        props2Help.put(HNMANADIR,"directory to write the harmnoics plus noise analysis results for each wav file");
-        
-        props2Help.put("HnmTimelineMaker.noiseModel", "Noise model: 1=WAVEFORM, 2=LPC, Default=1");
-        props2Help.put("HnmTimelineMaker.numFiltStages", "Number of filtering stages to smooth out maximum frequency of voicing curve: Range={0, 1, 2, 3, 4, 5}. 0 means no smoothing. Default=2");
-        props2Help.put("HnmTimelineMaker.medianFiltLen", "Length of median filter for smoothing the maximum frequency of voicing  curve: Range={4, 8, 12, 16, 20}, Default=12");
-        props2Help.put("HnmTimelineMaker.maFiltLen", "Length of moving average filter for smoothing the maximum frequency of voicing  curve: Range={4, 8, 12, 16, 20}, Default=12");
-        props2Help.put("HnmTimelineMaker.cumAmpTh", "Cumulative amplitude threshold [linear scale] for harmonic band voicing detection; decrease to increase max. freq. of voicing values. Range=[0.1, 10.0], Default=2.0");
-        props2Help.put("HnmTimelineMaker.maxAmpTh", "Maximum amplitude threshold [in DB] for harmonic band voicing detection. Decrease to increase max. freq. of voicing values. Range=[0.0, 20.0], Default=13.0");
-        props2Help.put("HnmTimelineMaker.harmDevPercent", "Percent deviation allowed for harmonic peak. Increase to increase max. freq. of voicing values. Range=[0.0, 100.0], Default=20.0");
-        props2Help.put("HnmTimelineMaker.sharpPeakAmpDiff", "Minimum amplitude difference [in DB] to declare an isolated peak as harmonic. Decrease to increase max. freq. of voicing values. range=[0.0, 20.0], Default=12.0");
-        props2Help.put("HnmTimelineMaker.minHarmonics", "Minimum total harmonics allowed in voiced regions. Range=[0, 100], Default=0");
-        props2Help.put("HnmTimelineMaker.maxHarmonics", "Maximum total harmonics allowed in voiced regions. Range=[minHarmonics, 100], Default=100");
-        props2Help.put("HnmTimelineMaker.minVoicedFreq", "Minimum voiced frequency for voiced regions [in Hz]. Range=[0.0, 0.5*samplingRate], Default=0");
-        props2Help.put("HnmTimelineMaker.maxVoicedFreq", "Maximum voiced frequency for voiced regions [in Hz]. Range=Default=[minVoicedFreq, 0.5*samplingRate], Default=5000");
-        props2Help.put("HnmTimelineMaker.maxFreqVoicingFinalShift", "Final amount of shift to be applied to the MWF curve [in Hz]. Range=[0.0, 0.5*samplingRate-maxVoicedFreq], Default=0");
-        props2Help.put("HnmTimelineMaker.neighsPercent", "Percentage of samples that the harmonic peak needs to be larger than within a band. Decrease to increase max. freq. of voicing values. Range=[0.0, 100.0], Default=50");
-        props2Help.put("HnmTimelineMaker.harmCepsOrder", "Cepstrum order to represent harmonic amplitudes. Increase to obtain better match with actual harmonic values. Range=[8, 40], Default=24");
-        props2Help.put("HnmTimelineMaker.regCepWarpMethod", "Warping method for regularized cepstrum estimation. 1=POST_MEL, 2=PRE_BARK, Default=1");
-        props2Help.put("HnmTimelineMaker.regCepsLambda", "Regularization term for cepstrum estimation. Increase to obtain smoother spectral match for harmonic amplitudes. However, this reduces the match with the actual amplitudes. Range=[0.0, 0.1], Default=1.0e-5");
-        props2Help.put("HnmTimelineMaker.noiseLpOrder", "Linear prediction order for LPC noise part. Range=[8, 50], Default=12");
-        props2Help.put("HnmTimelineMaker.preCoefNoise", "Pre-emphasis coefficient for linear prediction analysis of noise part. Range=[0.0, 0.99], Default=0.97");
-        props2Help.put("HnmTimelineMaker.hpfBeforeNoiseAnalysis", "Remove lowpass frequency residual after harmonic subtraction? 0=NO, 1=YES, Default=1");
-        props2Help.put("HnmTimelineMaker.harmNumPer", "Total periods for harmonic analysis. Range=[2.0, 4.0], Default=2");
-    }
-    
-    /**
-     *  Performs HNM analysis and writes the results to a single timeline file
-     *
-     */
-    public boolean compute()
-    {
-        long start = System.currentTimeMillis(); // start timing
-        
-        System.out.println("---- Importing Harmonics plus noise parameters\n\n");
-        System.out.println("Base directory: " + db.getProp(db.ROOTDIR) + "\n");
-        
-        /* Export the basename list into an array of strings */
-        String[] baseNameArray = bnl.getListAsArray();
-        
-        /* Prepare the output directory for the timelines if it does not exist */
-        File timelineDir = new File(db.getProp(db.FILEDIR));
-        
-        File ptcDir = new File(db.getProp(db.PTCDIR));
-        if (!ptcDir.exists()) {
-            ptcDir.mkdir();
-        }
-        
-        File hnaDir = new File(getProp(HNMANADIR));
-        if (!hnaDir.exists()) {
-            hnaDir.mkdir();
-        }
-        
-        try{
-            /* 1) Determine the reference sampling rate as being the sample rate of the first encountered
-             *    wav file */
-            WavReader wav = new WavReader(db.getProp(db.WAVDIR) + baseNameArray[0] + db.getProp(db.WAVEXT));
-            int globSampleRate = wav.getSampleRate();
-            System.out.println("---- Detected a global sample rate of: [" + globSampleRate + "] Hz." );
+	public final String PMDIR = "db.pmDir";
+	public final String PMEXT = "db.pmExtension";
 
-            System.out.println("---- Performing HNM analysis..." );
-            
-            /* Make the file name */
-            System.out.println( "Will create the hnm timeline in file [" 
-                    + getProp(HNMTIMELINE) + "]." );
-            
-            /* An example of processing header: */
-            Properties headerProps = new Properties();
+	public String getName() {
+		return "HnmTimelineMaker";
+	}
 
-            headerProps.setProperty("hnm.noiseModel", props.get("HnmTimelineMaker.noiseModel"));
-            headerProps.setProperty("hnm.numFiltStages", props.get("HnmTimelineMaker.numFiltStages"));
-            headerProps.setProperty("hnm.medianFiltLen", props.get("HnmTimelineMaker.medianFiltLen"));
-            headerProps.setProperty("hnm.maFiltLen", props.get("HnmTimelineMaker.maFiltLen"));
-            headerProps.setProperty("hnm.cumAmpTh", props.get("HnmTimelineMaker.cumAmpTh"));
-            headerProps.setProperty("hnm.maxAmpTh", props.get("HnmTimelineMaker.maxAmpTh"));
-            headerProps.setProperty("hnm.harmDevPercent", props.get("HnmTimelineMaker.harmDevPercent"));
-            headerProps.setProperty("hnm.sharpPeakAmpDiff", props.get("HnmTimelineMaker.sharpPeakAmpDiff"));
-            headerProps.setProperty("hnm.minHarmonics", props.get("HnmTimelineMaker.minHarmonics"));
-            headerProps.setProperty("hnm.maxHarmonics", props.get("HnmTimelineMaker.maxHarmonics"));
-            headerProps.setProperty("hnm.minVoicedFreq", props.get("HnmTimelineMaker.minVoicedFreq"));
-            headerProps.setProperty("hnm.maxVoicedFreq", props.get("HnmTimelineMaker.maxVoicedFreq"));
-            headerProps.setProperty("hnm.maxFreqVoicingFinalShift", props.get("HnmTimelineMaker.maxFreqVoicingFinalShift"));
-            headerProps.setProperty("hnm.neighsPercent", props.get("HnmTimelineMaker.neighsPercent"));
-            headerProps.setProperty("hnm.harmCepsOrder", props.get("HnmTimelineMaker.harmCepsOrder"));
-            headerProps.setProperty("hnm.regCepWarpMethod", props.get("HnmTimelineMaker.regCepWarpMethod"));
-            headerProps.setProperty("hnm.regCepsLambda", props.get("HnmTimelineMaker.regCepsLambda"));
-            headerProps.setProperty("hnm.noiseLpOrder", props.get("HnmTimelineMaker.noiseLpOrder"));
-            headerProps.setProperty("hnm.preCoefNoise", props.get("HnmTimelineMaker.preCoefNoise"));
-            headerProps.setProperty("hnm.hpfBeforeNoiseAnalysis", props.get("HnmTimelineMaker.hpfBeforeNoiseAnalysis"));
-            headerProps.setProperty("hnm.harmNumPer", props.get("HnmTimelineMaker.harmNumPer"));
+	public SortedMap<String, String> getDefaultProps(DatabaseLayout theDb) {
+		this.db = theDb;
+		if (props == null) {
+			HntmAnalyzerParams analysisParams = new HntmAnalyzerParams();
+			HntmSynthesizerParams synthesisParams = new HntmSynthesizerParams();
 
-            ByteArrayOutputStream baos = new ByteArrayOutputStream();
-            headerProps.store(baos, null);
-            String processingHeader = baos.toString("latin1");
-            
-            /* Instantiate the TimelineWriter: */
-            TimelineWriter hnmTimeline = new TimelineWriter( getProp(HNMTIMELINE), processingHeader, globSampleRate, 0.1 );
-            
-            //TO DO: Update these paratemers according to props
-            HntmAnalyzerParams analysisParams = new HntmAnalyzerParams();
-            HntmSynthesizerParams synthesisParamsBeforeNoiseAnalysis = new HntmSynthesizerParams();
-            
-            analysisParams.noiseModel = Integer.valueOf(props.get("HnmTimelineMaker.noiseModel"));
-            analysisParams.hnmPitchVoicingAnalyzerParams.numFilteringStages = Integer.valueOf(props.get("HnmTimelineMaker.numFiltStages"));
-            analysisParams.hnmPitchVoicingAnalyzerParams.medianFilterLength = Integer.valueOf(props.get("HnmTimelineMaker.medianFiltLen"));
-            analysisParams.hnmPitchVoicingAnalyzerParams.movingAverageFilterLength = Integer.valueOf(props.get("HnmTimelineMaker.maFiltLen"));
-            analysisParams.hnmPitchVoicingAnalyzerParams.cumulativeAmpThreshold = Float.valueOf(props.get("HnmTimelineMaker.cumAmpTh"));
-            analysisParams.hnmPitchVoicingAnalyzerParams.maximumAmpThresholdInDB = Float.valueOf(props.get("HnmTimelineMaker.maxAmpTh"));
-            analysisParams.hnmPitchVoicingAnalyzerParams.harmonicDeviationPercent = Float.valueOf(props.get("HnmTimelineMaker.harmDevPercent"));
-            analysisParams.hnmPitchVoicingAnalyzerParams.sharpPeakAmpDiffInDB = Float.valueOf(props.get("HnmTimelineMaker.sharpPeakAmpDiff"));
-            analysisParams.hnmPitchVoicingAnalyzerParams.minimumTotalHarmonics = Integer.valueOf(props.get("HnmTimelineMaker.minHarmonics"));
-            analysisParams.hnmPitchVoicingAnalyzerParams.maximumTotalHarmonics = Integer.valueOf(props.get("HnmTimelineMaker.maxHarmonics"));
-            analysisParams.hnmPitchVoicingAnalyzerParams.minimumVoicedFrequencyOfVoicing = Float.valueOf(props.get("HnmTimelineMaker.minVoicedFreq"));
-            analysisParams.hnmPitchVoicingAnalyzerParams.maximumVoicedFrequencyOfVoicing = Float.valueOf(props.get("HnmTimelineMaker.maxVoicedFreq"));
-            analysisParams.hnmPitchVoicingAnalyzerParams.maximumFrequencyOfVoicingFinalShift = Float.valueOf(props.get("HnmTimelineMaker.maxFreqVoicingFinalShift"));
-            analysisParams.hnmPitchVoicingAnalyzerParams.neighsPercent = Float.valueOf(props.get("HnmTimelineMaker.neighsPercent"));
-            analysisParams.harmonicPartCepstrumOrder = Integer.valueOf(props.get("HnmTimelineMaker.harmCepsOrder"));
-            analysisParams.regularizedCepstrumWarpingMethod = Integer.valueOf(props.get("HnmTimelineMaker.regCepWarpMethod"));
-            analysisParams.regularizedCepstrumLambdaHarmonic = Float.valueOf(props.get("HnmTimelineMaker.regCepsLambda"));
-            analysisParams.noisePartLpOrder = Integer.valueOf(props.get("HnmTimelineMaker.noiseLpOrder"));
-            analysisParams.preemphasisCoefNoise = Float.valueOf(props.get("HnmTimelineMaker.preCoefNoise"));
-            analysisParams.hpfBeforeNoiseAnalysis = Boolean.valueOf(props.get("HnmTimelineMaker.hpfBeforeNoiseAnalysis"));
-            analysisParams.numPeriodsHarmonicsExtraction = Float.valueOf(props.get("HnmTimelineMaker.harmNumPer"));
-            
-            analysisParams.isSilentAnalysis = true;
-            //
-            
-            /* 2) Write the datagrams and feed the index */
-            float totalDuration = 0.0f;  // Accumulator for the total timeline duration
-            long totalTime = 0l;
-            long numDatagrams = 0l; // Total number of hnm datagrams in the timeline file
-            
-            /* For each wav file: */
-            int n, i;
-            double f0WindowSizeInSeconds = 0;
-            double f0SkipSizeInSeconds = 0;
-            
-            for (n=0; n<baseNameArray.length; n++) 
-            {
-                percent = 100*n/baseNameArray.length;
-                /* - open+load */
-                System.out.println( baseNameArray[n] );
-                String wavFile = db.getProp(db.WAVDIR) + baseNameArray[n] + db.getProp(db.WAVEXT); 
-                
-                ESTTrackReader pmFile = new ESTTrackReader( db.getProp(PMDIR) + baseNameArray[n] + db.getProp(PMEXT));
-                totalDuration += pmFile.getTimeSpan();
-        
-                HntmAnalyzer ha = new HntmAnalyzer();
-                String hnmAnalysisFile = getProp(HNMANADIR) + baseNameArray[n] + hnmAnalysisFileExt; 
-                
-                HntmSpeechSignal hnmSignal = null;
-                if (FileUtils.exists(hnmAnalysisFile))
-                    hnmSignal = new HntmSpeechSignal(hnmAnalysisFile, analysisParams.noiseModel);
-                else
-                {
-                    wav = new WavReader(wavFile);
-                    short[] wave = wav.getSamples();
-                    
-                    String ptcFile = db.getProp(db.PTCDIR) + baseNameArray[n] + db.getProp(db.PTCEXT);
-                    PitchReaderWriter f0 = null;
-                    if (FileUtils.exists(ptcFile))
-                        f0 = new PitchReaderWriter(ptcFile);
-                    else
-                    {
-                        PitchFileHeader pitchDetectorParams = new PitchFileHeader();
-                        // default values are problematic; for now, re-use the parameters from PraatPitchmarker:
-                        pitchDetectorParams.minimumF0 = Double.parseDouble(db.getProperty("PraatPitchmarker.minPitch"));
-                        pitchDetectorParams.maximumF0 = Double.parseDouble(db.getProperty("PraatPitchmarker.maxPitch"));
-                        F0TrackerAutocorrelationHeuristic pitchDetector = new F0TrackerAutocorrelationHeuristic(pitchDetectorParams);
-                        f0 = pitchDetector.pitchAnalyzeWavFile(wavFile, ptcFile);
-                    }
-                    
-                    int frameStart = 0;
-                    int frameEnd = 0;
-                    long duration;
+			props = new TreeMap<String, String>();
 
-                    for (i=0; i<pmFile.getNumFrames()-1; i++ ) 
-                    {
-                        frameStart = (int)( (double)pmFile.getTime(i) * (double)(globSampleRate) );
-                        frameEnd = (int)( (double)pmFile.getTime(i+1) * (double)(globSampleRate) );
-                        assert frameEnd <= wave.length : "Frame ends after end of wave data: " + frameEnd + " > " + wave.length;
-                        duration = frameEnd - frameStart;
-                        if (duration<5)
-                            System.out.println("Too short duration");
-                    }   
-                    
-                    PitchMarks pm = new PitchMarks(pmFile, globSampleRate);
-                    pm.findAndSetUnvoicedF0s(f0.contour, f0.header, globSampleRate);
-                   
-                    if (n==0)
-                    {
-                        f0WindowSizeInSeconds = f0.header.windowSizeInSeconds;
-                        if (pmFile.getNumFrames()>1.0)
-                            f0SkipSizeInSeconds = SignalProcUtils.sampleFloat2time(((float)wave.length-SignalProcUtils.time2sample(f0WindowSizeInSeconds, globSampleRate))/(pmFile.getNumFrames()-1.0f), globSampleRate);
-                        else
-                            f0SkipSizeInSeconds = f0.header.skipSizeInSeconds;
-                    }
-                    
-                    //Use pitch marks from pm folder
-                    hnmSignal = ha.analyze(wave, wav.getSampleRate(), pm, f0WindowSizeInSeconds, f0SkipSizeInSeconds, pm.f0s, null, analysisParams, synthesisParamsBeforeNoiseAnalysis, hnmAnalysisFile); 
-                    
-                    //Use autocorrelation pitch detector based pitch marks
-                    //hnmSignal = ha.analyze(wave, wav.getSampleRate(), f0, null, analysisParams, synthesisParamsBeforeNoiseAnalysis, hnmAnalysisFile); 
-                    
-                    float tAnalysisInSeconds = hnmSignal.frames[0].deltaAnalysisTimeInSeconds;
-                    for (i=0; i<hnmSignal.frames.length; i++ ) 
-                    {
-                        frameStart = frameEnd;
-                        frameEnd = SignalProcUtils.time2sample(tAnalysisInSeconds, hnmSignal.samplingRateInHz);
-                        
-                        assert frameEnd <= wave.length : "Frame ends after end of wave data: " + frameEnd + " > " + wave.length;
-                        duration = frameEnd - frameStart;
-                        
-                        tAnalysisInSeconds += hnmSignal.frames[i].deltaAnalysisTimeInSeconds;
-                    }
-                }
-                
-                /* - For each frame in the hnm modeled speech signal: */
-                int frameStart = 0;
-                int frameEnd = 0;
-                int duration = 0;
-                long localTime = 0l;
-                int currentIndex;
-                int prevIndex = -1;
-                float[] analysisTimes = hnmSignal.getAnalysisTimes();
-                float tAnalysisInSeconds;
+			props.put(HNMTIMELINE, db.getProp(db.FILEDIR) + "timeline_hnm" + db.getProp(db.MARYEXT));
 
-                float[] pmTimes = new float[pmFile.getNumFrames()];
+			props.put(HNMANADIR, db.getProp(db.ROOTDIR) + "hna" + System.getProperty("file.separator"));
 
-                for (i=0; i<pmFile.getNumFrames(); i++)
-                    pmTimes[i] = pmFile.getTime(i);
+			props.put("HnmTimelineMaker.noiseModel", String.valueOf(analysisParams.noiseModel));
+			props.put("HnmTimelineMaker.numFiltStages",
+					String.valueOf(analysisParams.hnmPitchVoicingAnalyzerParams.numFilteringStages));
+			props.put("HnmTimelineMaker.medianFiltLen",
+					String.valueOf(analysisParams.hnmPitchVoicingAnalyzerParams.medianFilterLength));
+			props.put("HnmTimelineMaker.maFiltLen",
+					String.valueOf(analysisParams.hnmPitchVoicingAnalyzerParams.movingAverageFilterLength));
+			props.put("HnmTimelineMaker.cumAmpTh",
+					String.valueOf(analysisParams.hnmPitchVoicingAnalyzerParams.cumulativeAmpThreshold));
+			props.put("HnmTimelineMaker.maxAmpTh",
+					String.valueOf(analysisParams.hnmPitchVoicingAnalyzerParams.maximumAmpThresholdInDB));
+			props.put("HnmTimelineMaker.harmDevPercent",
+					String.valueOf(analysisParams.hnmPitchVoicingAnalyzerParams.harmonicDeviationPercent));
+			props.put("HnmTimelineMaker.sharpPeakAmpDiff",
+					String.valueOf(analysisParams.hnmPitchVoicingAnalyzerParams.sharpPeakAmpDiffInDB));
+			props.put("HnmTimelineMaker.minHarmonics",
+					String.valueOf(analysisParams.hnmPitchVoicingAnalyzerParams.minimumTotalHarmonics));
+			props.put("HnmTimelineMaker.maxHarmonics",
+					String.valueOf(analysisParams.hnmPitchVoicingAnalyzerParams.maximumTotalHarmonics));
+			props.put("HnmTimelineMaker.minVoicedFreq",
+					String.valueOf(analysisParams.hnmPitchVoicingAnalyzerParams.minimumVoicedFrequencyOfVoicing));
+			props.put("HnmTimelineMaker.maxVoicedFreq",
+					String.valueOf(analysisParams.hnmPitchVoicingAnalyzerParams.maximumVoicedFrequencyOfVoicing));
+			props.put("HnmTimelineMaker.maxFreqVoicingFinalShift",
+					String.valueOf(analysisParams.hnmPitchVoicingAnalyzerParams.maximumFrequencyOfVoicingFinalShift));
+			props.put("HnmTimelineMaker.neighsPercent",
+					String.valueOf(analysisParams.hnmPitchVoicingAnalyzerParams.neighsPercent));
+			props.put("HnmTimelineMaker.harmCepsOrder", String.valueOf(analysisParams.harmonicPartCepstrumOrder));
+			props.put("HnmTimelineMaker.regCepWarpMethod", String.valueOf(analysisParams.regularizedCepstrumWarpingMethod));
+			props.put("HnmTimelineMaker.regCepsLambda", String.valueOf(analysisParams.regularizedCepstrumLambdaHarmonic));
+			props.put("HnmTimelineMaker.noiseLpOrder", String.valueOf(analysisParams.noisePartLpOrder));
+			props.put("HnmTimelineMaker.preCoefNoise", String.valueOf(analysisParams.preemphasisCoefNoise));
+			props.put("HnmTimelineMaker.hpfBeforeNoiseAnalysis", String.valueOf(analysisParams.hpfBeforeNoiseAnalysis));
+			props.put("HnmTimelineMaker.harmNumPer", String.valueOf(analysisParams.numPeriodsHarmonicsExtraction));
+		}
 
-                tAnalysisInSeconds = hnmSignal.frames[0].deltaAnalysisTimeInSeconds;
-                for (i=0; i<pmFile.getNumFrames(); i++)
-                {
-                    if (i<hnmSignal.frames.length)
-                    {
-                        frameStart = frameEnd;
-                        frameEnd = (int)( (double)pmFile.getTime(i) * (double)(globSampleRate) );
-                        duration = frameEnd - frameStart;
+		return props;
+	}
 
-                        if (frameEnd>0)
-                        {
-                            currentIndex = MathUtils.findClosest(analysisTimes, pmFile.getTime(i));
+	protected void setupHelp() {
+		props2Help = new TreeMap<String, String>();
 
-                            //System.out.println(currentIndex + " " + i);
+		props2Help.put(HNMTIMELINE, "file containing all hnm noise waveform files. Will be created by this module");
+		props2Help.put(HNMANADIR, "directory to write the harmnoics plus noise analysis results for each wav file");
 
-                            hnmSignal.frames[currentIndex].tAnalysisInSeconds = tAnalysisInSeconds;
+		props2Help.put("HnmTimelineMaker.noiseModel", "Noise model: 1=WAVEFORM, 2=LPC, Default=1");
+		props2Help
+				.put("HnmTimelineMaker.numFiltStages",
+						"Number of filtering stages to smooth out maximum frequency of voicing curve: Range={0, 1, 2, 3, 4, 5}. 0 means no smoothing. Default=2");
+		props2Help
+				.put("HnmTimelineMaker.medianFiltLen",
+						"Length of median filter for smoothing the maximum frequency of voicing  curve: Range={4, 8, 12, 16, 20}, Default=12");
+		props2Help
+				.put("HnmTimelineMaker.maFiltLen",
+						"Length of moving average filter for smoothing the maximum frequency of voicing  curve: Range={4, 8, 12, 16, 20}, Default=12");
+		props2Help
+				.put("HnmTimelineMaker.cumAmpTh",
+						"Cumulative amplitude threshold [linear scale] for harmonic band voicing detection; decrease to increase max. freq. of voicing values. Range=[0.1, 10.0], Default=2.0");
+		props2Help
+				.put("HnmTimelineMaker.maxAmpTh",
+						"Maximum amplitude threshold [in DB] for harmonic band voicing detection. Decrease to increase max. freq. of voicing values. Range=[0.0, 20.0], Default=13.0");
+		props2Help
+				.put("HnmTimelineMaker.harmDevPercent",
+						"Percent deviation allowed for harmonic peak. Increase to increase max. freq. of voicing values. Range=[0.0, 100.0], Default=20.0");
+		props2Help
+				.put("HnmTimelineMaker.sharpPeakAmpDiff",
+						"Minimum amplitude difference [in DB] to declare an isolated peak as harmonic. Decrease to increase max. freq. of voicing values. range=[0.0, 20.0], Default=12.0");
+		props2Help.put("HnmTimelineMaker.minHarmonics",
+				"Minimum total harmonics allowed in voiced regions. Range=[0, 100], Default=0");
+		props2Help.put("HnmTimelineMaker.maxHarmonics",
+				"Maximum total harmonics allowed in voiced regions. Range=[minHarmonics, 100], Default=100");
+		props2Help.put("HnmTimelineMaker.minVoicedFreq",
+				"Minimum voiced frequency for voiced regions [in Hz]. Range=[0.0, 0.5*samplingRate], Default=0");
+		props2Help
+				.put("HnmTimelineMaker.maxVoicedFreq",
+						"Maximum voiced frequency for voiced regions [in Hz]. Range=Default=[minVoicedFreq, 0.5*samplingRate], Default=5000");
+		props2Help
+				.put("HnmTimelineMaker.maxFreqVoicingFinalShift",
+						"Final amount of shift to be applied to the MWF curve [in Hz]. Range=[0.0, 0.5*samplingRate-maxVoicedFreq], Default=0");
+		props2Help
+				.put("HnmTimelineMaker.neighsPercent",
+						"Percentage of samples that the harmonic peak needs to be larger than within a band. Decrease to increase max. freq. of voicing values. Range=[0.0, 100.0], Default=50");
+		props2Help
+				.put("HnmTimelineMaker.harmCepsOrder",
+						"Cepstrum order to represent harmonic amplitudes. Increase to obtain better match with actual harmonic values. Range=[8, 40], Default=24");
+		props2Help.put("HnmTimelineMaker.regCepWarpMethod",
+				"Warping method for regularized cepstrum estimation. 1=POST_MEL, 2=PRE_BARK, Default=1");
+		props2Help
+				.put("HnmTimelineMaker.regCepsLambda",
+						"Regularization term for cepstrum estimation. Increase to obtain smoother spectral match for harmonic amplitudes. However, this reduces the match with the actual amplitudes. Range=[0.0, 0.1], Default=1.0e-5");
+		props2Help.put("HnmTimelineMaker.noiseLpOrder", "Linear prediction order for LPC noise part. Range=[8, 50], Default=12");
+		props2Help.put("HnmTimelineMaker.preCoefNoise",
+				"Pre-emphasis coefficient for linear prediction analysis of noise part. Range=[0.0, 0.99], Default=0.97");
+		props2Help.put("HnmTimelineMaker.hpfBeforeNoiseAnalysis",
+				"Remove lowpass frequency residual after harmonic subtraction? 0=NO, 1=YES, Default=1");
+		props2Help.put("HnmTimelineMaker.harmNumPer", "Total periods for harmonic analysis. Range=[2.0, 4.0], Default=2");
+	}
 
-                            // Feed the datagram to the timeline
-                            hnmTimeline.feed( new HnmDatagram(duration, hnmSignal.frames[currentIndex]) , globSampleRate );
-                            totalTime += duration;
-                            localTime += duration;
-                            tAnalysisInSeconds += hnmSignal.frames[currentIndex].deltaAnalysisTimeInSeconds;
-                        }
-                    }
-                }
-                
-                /*
-                tAnalysisInSeconds = hnmSignal.frames[0].deltaAnalysisTimeInSeconds;
-                for (i=0; i<hnmSignal.frames.length; i++)
-                {
-                    frameStart = frameEnd;   
-                    frameEnd = SignalProcUtils.time2sample(tAnalysisInSeconds, hnmSignal.samplingRateInHz);
-                    duration = frameEnd - frameStart;
+	/**
+	 * Performs HNM analysis and writes the results to a single timeline file
+	 *
+	 */
+	public boolean compute() {
+		long start = System.currentTimeMillis(); // start timing
 
-                    // Feed the datagram to the timeline
-                    hnmTimeline.feed( new HnmDatagram(duration, hnmSignal.frames[i]) , globSampleRate );
-                    totalTime += duration;
-                    localTime += duration;
-                    
-                    tAnalysisInSeconds += hnmSignal.frames[i].deltaAnalysisTimeInSeconds;
-                }
-                */
+		System.out.println("---- Importing Harmonics plus noise parameters\n\n");
+		System.out.println("Base directory: " + db.getProp(db.ROOTDIR) + "\n");
 
-                System.out.println(String.valueOf(n+1) + " of " + String.valueOf(baseNameArray.length) + " done...");
-                numDatagrams += hnmSignal.frames.length;
-            }
-            hnmTimeline.close();
-            
-            System.out.println("---- Done." );
-            
-            /* 7) Print some stats and close the file */
-            System.out.println( "---- hnm timeline result:");
-            System.out.println( "Number of files scanned: " + baseNameArray.length );
-            System.out.println( "Total duration: [" + totalTime + "] samples / [" + ((double)(totalTime) / (double)(globSampleRate)) + "] seconds." );
-            System.out.println( "Number of frames: [" + numDatagrams + "]." );
-            System.out.println( "Size of the index: [" + hnmTimeline.getIndex().getNumIdx() + "] ("
-                    + (hnmTimeline.getIndex().getNumIdx() * 16) + " bytes, i.e. "
-                    + new DecimalFormat("#.##").format((double)(hnmTimeline.getIndex().getNumIdx()) * 16.0 / 1048576.0) + " megs).");
-            
-            long stop = System.currentTimeMillis(); // stop timing
-            System.out.println("The process took " + (stop - start)/(1000.0*60) + " minutes to complete..."); // print execution time
-            
-            System.out.println( "---- hnm timeline done.");
-            
-        }
-        catch ( SecurityException e ) {
-            System.err.println( "Error: you don't have write access to the target database directory." );
-        }
-        catch (Exception e) {
-            e.printStackTrace();
-            System.err.println(e);
-        }
-        
-        return( true );
-    }
-    
-    /**
-     * Provide the progress of computation, in percent, or -1 if
-     * that feature is not implemented.
-     * @return -1 if not implemented, or an integer between 0 and 100.
-     */
-    public int getProgress()
-    {
-        return percent;
-    }
-    
-    public static void main(String[] args) throws Exception
-    {        
-        VoiceImportComponent vic  =  new HnmTimelineMaker();
-        DatabaseLayout db = new DatabaseLayout(vic);
-        vic.compute();
-    }
+		/* Export the basename list into an array of strings */
+		String[] baseNameArray = bnl.getListAsArray();
+
+		/* Prepare the output directory for the timelines if it does not exist */
+		File timelineDir = new File(db.getProp(db.FILEDIR));
+
+		File ptcDir = new File(db.getProp(db.PTCDIR));
+		if (!ptcDir.exists()) {
+			ptcDir.mkdir();
+		}
+
+		File hnaDir = new File(getProp(HNMANADIR));
+		if (!hnaDir.exists()) {
+			hnaDir.mkdir();
+		}
+
+		try {
+			/*
+			 * 1) Determine the reference sampling rate as being the sample rate of the first encountered wav file
+			 */
+			WavReader wav = new WavReader(db.getProp(db.WAVDIR) + baseNameArray[0] + db.getProp(db.WAVEXT));
+			int globSampleRate = wav.getSampleRate();
+			System.out.println("---- Detected a global sample rate of: [" + globSampleRate + "] Hz.");
+
+			System.out.println("---- Performing HNM analysis...");
+
+			/* Make the file name */
+			System.out.println("Will create the hnm timeline in file [" + getProp(HNMTIMELINE) + "].");
+
+			/* An example of processing header: */
+			Properties headerProps = new Properties();
+
+			headerProps.setProperty("hnm.noiseModel", props.get("HnmTimelineMaker.noiseModel"));
+			headerProps.setProperty("hnm.numFiltStages", props.get("HnmTimelineMaker.numFiltStages"));
+			headerProps.setProperty("hnm.medianFiltLen", props.get("HnmTimelineMaker.medianFiltLen"));
+			headerProps.setProperty("hnm.maFiltLen", props.get("HnmTimelineMaker.maFiltLen"));
+			headerProps.setProperty("hnm.cumAmpTh", props.get("HnmTimelineMaker.cumAmpTh"));
+			headerProps.setProperty("hnm.maxAmpTh", props.get("HnmTimelineMaker.maxAmpTh"));
+			headerProps.setProperty("hnm.harmDevPercent", props.get("HnmTimelineMaker.harmDevPercent"));
+			headerProps.setProperty("hnm.sharpPeakAmpDiff", props.get("HnmTimelineMaker.sharpPeakAmpDiff"));
+			headerProps.setProperty("hnm.minHarmonics", props.get("HnmTimelineMaker.minHarmonics"));
+			headerProps.setProperty("hnm.maxHarmonics", props.get("HnmTimelineMaker.maxHarmonics"));
+			headerProps.setProperty("hnm.minVoicedFreq", props.get("HnmTimelineMaker.minVoicedFreq"));
+			headerProps.setProperty("hnm.maxVoicedFreq", props.get("HnmTimelineMaker.maxVoicedFreq"));
+			headerProps.setProperty("hnm.maxFreqVoicingFinalShift", props.get("HnmTimelineMaker.maxFreqVoicingFinalShift"));
+			headerProps.setProperty("hnm.neighsPercent", props.get("HnmTimelineMaker.neighsPercent"));
+			headerProps.setProperty("hnm.harmCepsOrder", props.get("HnmTimelineMaker.harmCepsOrder"));
+			headerProps.setProperty("hnm.regCepWarpMethod", props.get("HnmTimelineMaker.regCepWarpMethod"));
+			headerProps.setProperty("hnm.regCepsLambda", props.get("HnmTimelineMaker.regCepsLambda"));
+			headerProps.setProperty("hnm.noiseLpOrder", props.get("HnmTimelineMaker.noiseLpOrder"));
+			headerProps.setProperty("hnm.preCoefNoise", props.get("HnmTimelineMaker.preCoefNoise"));
+			headerProps.setProperty("hnm.hpfBeforeNoiseAnalysis", props.get("HnmTimelineMaker.hpfBeforeNoiseAnalysis"));
+			headerProps.setProperty("hnm.harmNumPer", props.get("HnmTimelineMaker.harmNumPer"));
+
+			ByteArrayOutputStream baos = new ByteArrayOutputStream();
+			headerProps.store(baos, null);
+			String processingHeader = baos.toString("latin1");
+
+			/* Instantiate the TimelineWriter: */
+			TimelineWriter hnmTimeline = new TimelineWriter(getProp(HNMTIMELINE), processingHeader, globSampleRate, 0.1);
+
+			// TO DO: Update these paratemers according to props
+			HntmAnalyzerParams analysisParams = new HntmAnalyzerParams();
+			HntmSynthesizerParams synthesisParamsBeforeNoiseAnalysis = new HntmSynthesizerParams();
+
+			analysisParams.noiseModel = Integer.valueOf(props.get("HnmTimelineMaker.noiseModel"));
+			analysisParams.hnmPitchVoicingAnalyzerParams.numFilteringStages = Integer.valueOf(props
+					.get("HnmTimelineMaker.numFiltStages"));
+			analysisParams.hnmPitchVoicingAnalyzerParams.medianFilterLength = Integer.valueOf(props
+					.get("HnmTimelineMaker.medianFiltLen"));
+			analysisParams.hnmPitchVoicingAnalyzerParams.movingAverageFilterLength = Integer.valueOf(props
+					.get("HnmTimelineMaker.maFiltLen"));
+			analysisParams.hnmPitchVoicingAnalyzerParams.cumulativeAmpThreshold = Float.valueOf(props
+					.get("HnmTimelineMaker.cumAmpTh"));
+			analysisParams.hnmPitchVoicingAnalyzerParams.maximumAmpThresholdInDB = Float.valueOf(props
+					.get("HnmTimelineMaker.maxAmpTh"));
+			analysisParams.hnmPitchVoicingAnalyzerParams.harmonicDeviationPercent = Float.valueOf(props
+					.get("HnmTimelineMaker.harmDevPercent"));
+			analysisParams.hnmPitchVoicingAnalyzerParams.sharpPeakAmpDiffInDB = Float.valueOf(props
+					.get("HnmTimelineMaker.sharpPeakAmpDiff"));
+			analysisParams.hnmPitchVoicingAnalyzerParams.minimumTotalHarmonics = Integer.valueOf(props
+					.get("HnmTimelineMaker.minHarmonics"));
+			analysisParams.hnmPitchVoicingAnalyzerParams.maximumTotalHarmonics = Integer.valueOf(props
+					.get("HnmTimelineMaker.maxHarmonics"));
+			analysisParams.hnmPitchVoicingAnalyzerParams.minimumVoicedFrequencyOfVoicing = Float.valueOf(props
+					.get("HnmTimelineMaker.minVoicedFreq"));
+			analysisParams.hnmPitchVoicingAnalyzerParams.maximumVoicedFrequencyOfVoicing = Float.valueOf(props
+					.get("HnmTimelineMaker.maxVoicedFreq"));
+			analysisParams.hnmPitchVoicingAnalyzerParams.maximumFrequencyOfVoicingFinalShift = Float.valueOf(props
+					.get("HnmTimelineMaker.maxFreqVoicingFinalShift"));
+			analysisParams.hnmPitchVoicingAnalyzerParams.neighsPercent = Float.valueOf(props
+					.get("HnmTimelineMaker.neighsPercent"));
+			analysisParams.harmonicPartCepstrumOrder = Integer.valueOf(props.get("HnmTimelineMaker.harmCepsOrder"));
+			analysisParams.regularizedCepstrumWarpingMethod = Integer.valueOf(props.get("HnmTimelineMaker.regCepWarpMethod"));
+			analysisParams.regularizedCepstrumLambdaHarmonic = Float.valueOf(props.get("HnmTimelineMaker.regCepsLambda"));
+			analysisParams.noisePartLpOrder = Integer.valueOf(props.get("HnmTimelineMaker.noiseLpOrder"));
+			analysisParams.preemphasisCoefNoise = Float.valueOf(props.get("HnmTimelineMaker.preCoefNoise"));
+			analysisParams.hpfBeforeNoiseAnalysis = Boolean.valueOf(props.get("HnmTimelineMaker.hpfBeforeNoiseAnalysis"));
+			analysisParams.numPeriodsHarmonicsExtraction = Float.valueOf(props.get("HnmTimelineMaker.harmNumPer"));
+
+			analysisParams.isSilentAnalysis = true;
+			//
+
+			/* 2) Write the datagrams and feed the index */
+			float totalDuration = 0.0f; // Accumulator for the total timeline duration
+			long totalTime = 0l;
+			long numDatagrams = 0l; // Total number of hnm datagrams in the timeline file
+
+			/* For each wav file: */
+			int n, i;
+			double f0WindowSizeInSeconds = 0;
+			double f0SkipSizeInSeconds = 0;
+
+			for (n = 0; n < baseNameArray.length; n++) {
+				percent = 100 * n / baseNameArray.length;
+				/* - open+load */
+				System.out.println(baseNameArray[n]);
+				String wavFile = db.getProp(db.WAVDIR) + baseNameArray[n] + db.getProp(db.WAVEXT);
+
+				ESTTrackReader pmFile = new ESTTrackReader(db.getProp(PMDIR) + baseNameArray[n] + db.getProp(PMEXT));
+				totalDuration += pmFile.getTimeSpan();
+
+				HntmAnalyzer ha = new HntmAnalyzer();
+				String hnmAnalysisFile = getProp(HNMANADIR) + baseNameArray[n] + hnmAnalysisFileExt;
+
+				HntmSpeechSignal hnmSignal = null;
+				if (FileUtils.exists(hnmAnalysisFile))
+					hnmSignal = new HntmSpeechSignal(hnmAnalysisFile, analysisParams.noiseModel);
+				else {
+					wav = new WavReader(wavFile);
+					short[] wave = wav.getSamples();
+
+					String ptcFile = db.getProp(db.PTCDIR) + baseNameArray[n] + db.getProp(db.PTCEXT);
+					PitchReaderWriter f0 = null;
+					if (FileUtils.exists(ptcFile))
+						f0 = new PitchReaderWriter(ptcFile);
+					else {
+						PitchFileHeader pitchDetectorParams = new PitchFileHeader();
+						// default values are problematic; for now, re-use the parameters from PraatPitchmarker:
+						pitchDetectorParams.minimumF0 = Double.parseDouble(db.getProperty("PraatPitchmarker.minPitch"));
+						pitchDetectorParams.maximumF0 = Double.parseDouble(db.getProperty("PraatPitchmarker.maxPitch"));
+						F0TrackerAutocorrelationHeuristic pitchDetector = new F0TrackerAutocorrelationHeuristic(
+								pitchDetectorParams);
+						f0 = pitchDetector.pitchAnalyzeWavFile(wavFile, ptcFile);
+					}
+
+					int frameStart = 0;
+					int frameEnd = 0;
+					long duration;
+
+					for (i = 0; i < pmFile.getNumFrames() - 1; i++) {
+						frameStart = (int) ((double) pmFile.getTime(i) * (double) (globSampleRate));
+						frameEnd = (int) ((double) pmFile.getTime(i + 1) * (double) (globSampleRate));
+						assert frameEnd <= wave.length : "Frame ends after end of wave data: " + frameEnd + " > " + wave.length;
+						duration = frameEnd - frameStart;
+						if (duration < 5)
+							System.out.println("Too short duration");
+					}
+
+					PitchMarks pm = new PitchMarks(pmFile, globSampleRate);
+					pm.findAndSetUnvoicedF0s(f0.contour, f0.header, globSampleRate);
+
+					if (n == 0) {
+						f0WindowSizeInSeconds = f0.header.windowSizeInSeconds;
+						if (pmFile.getNumFrames() > 1.0)
+							f0SkipSizeInSeconds = SignalProcUtils.sampleFloat2time(
+									((float) wave.length - SignalProcUtils.time2sample(f0WindowSizeInSeconds, globSampleRate))
+											/ (pmFile.getNumFrames() - 1.0f), globSampleRate);
+						else
+							f0SkipSizeInSeconds = f0.header.skipSizeInSeconds;
+					}
+
+					// Use pitch marks from pm folder
+					hnmSignal = ha.analyze(wave, wav.getSampleRate(), pm, f0WindowSizeInSeconds, f0SkipSizeInSeconds, pm.f0s,
+							null, analysisParams, synthesisParamsBeforeNoiseAnalysis, hnmAnalysisFile);
+
+					// Use autocorrelation pitch detector based pitch marks
+					// hnmSignal = ha.analyze(wave, wav.getSampleRate(), f0, null, analysisParams,
+					// synthesisParamsBeforeNoiseAnalysis, hnmAnalysisFile);
+
+					float tAnalysisInSeconds = hnmSignal.frames[0].deltaAnalysisTimeInSeconds;
+					for (i = 0; i < hnmSignal.frames.length; i++) {
+						frameStart = frameEnd;
+						frameEnd = SignalProcUtils.time2sample(tAnalysisInSeconds, hnmSignal.samplingRateInHz);
+
+						assert frameEnd <= wave.length : "Frame ends after end of wave data: " + frameEnd + " > " + wave.length;
+						duration = frameEnd - frameStart;
+
+						tAnalysisInSeconds += hnmSignal.frames[i].deltaAnalysisTimeInSeconds;
+					}
+				}
+
+				/* - For each frame in the hnm modeled speech signal: */
+				int frameStart = 0;
+				int frameEnd = 0;
+				int duration = 0;
+				long localTime = 0l;
+				int currentIndex;
+				int prevIndex = -1;
+				float[] analysisTimes = hnmSignal.getAnalysisTimes();
+				float tAnalysisInSeconds;
+
+				float[] pmTimes = new float[pmFile.getNumFrames()];
+
+				for (i = 0; i < pmFile.getNumFrames(); i++)
+					pmTimes[i] = pmFile.getTime(i);
+
+				tAnalysisInSeconds = hnmSignal.frames[0].deltaAnalysisTimeInSeconds;
+				for (i = 0; i < pmFile.getNumFrames(); i++) {
+					if (i < hnmSignal.frames.length) {
+						frameStart = frameEnd;
+						frameEnd = (int) ((double) pmFile.getTime(i) * (double) (globSampleRate));
+						duration = frameEnd - frameStart;
+
+						if (frameEnd > 0) {
+							currentIndex = MathUtils.findClosest(analysisTimes, pmFile.getTime(i));
+
+							// System.out.println(currentIndex + " " + i);
+
+							hnmSignal.frames[currentIndex].tAnalysisInSeconds = tAnalysisInSeconds;
+
+							// Feed the datagram to the timeline
+							hnmTimeline.feed(new HnmDatagram(duration, hnmSignal.frames[currentIndex]), globSampleRate);
+							totalTime += duration;
+							localTime += duration;
+							tAnalysisInSeconds += hnmSignal.frames[currentIndex].deltaAnalysisTimeInSeconds;
+						}
+					}
+				}
+
+				/*
+				 * tAnalysisInSeconds = hnmSignal.frames[0].deltaAnalysisTimeInSeconds; for (i=0; i<hnmSignal.frames.length; i++)
+				 * { frameStart = frameEnd; frameEnd = SignalProcUtils.time2sample(tAnalysisInSeconds,
+				 * hnmSignal.samplingRateInHz); duration = frameEnd - frameStart;
+				 * 
+				 * // Feed the datagram to the timeline hnmTimeline.feed( new HnmDatagram(duration, hnmSignal.frames[i]) ,
+				 * globSampleRate ); totalTime += duration; localTime += duration;
+				 * 
+				 * tAnalysisInSeconds += hnmSignal.frames[i].deltaAnalysisTimeInSeconds; }
+				 */
+
+				System.out.println(String.valueOf(n + 1) + " of " + String.valueOf(baseNameArray.length) + " done...");
+				numDatagrams += hnmSignal.frames.length;
+			}
+			hnmTimeline.close();
+
+			System.out.println("---- Done.");
+
+			/* 7) Print some stats and close the file */
+			System.out.println("---- hnm timeline result:");
+			System.out.println("Number of files scanned: " + baseNameArray.length);
+			System.out.println("Total duration: [" + totalTime + "] samples / ["
+					+ ((double) (totalTime) / (double) (globSampleRate)) + "] seconds.");
+			System.out.println("Number of frames: [" + numDatagrams + "].");
+			System.out.println("Size of the index: [" + hnmTimeline.getIndex().getNumIdx() + "] ("
+					+ (hnmTimeline.getIndex().getNumIdx() * 16) + " bytes, i.e. "
+					+ new DecimalFormat("#.##").format((double) (hnmTimeline.getIndex().getNumIdx()) * 16.0 / 1048576.0)
+					+ " megs).");
+
+			long stop = System.currentTimeMillis(); // stop timing
+			System.out.println("The process took " + (stop - start) / (1000.0 * 60) + " minutes to complete..."); // print
+																													// execution
+																													// time
+
+			System.out.println("---- hnm timeline done.");
+
+		} catch (SecurityException e) {
+			System.err.println("Error: you don't have write access to the target database directory.");
+		} catch (Exception e) {
+			e.printStackTrace();
+			System.err.println(e);
+		}
+
+		return (true);
+	}
+
+	/**
+	 * Provide the progress of computation, in percent, or -1 if that feature is not implemented.
+	 * 
+	 * @return -1 if not implemented, or an integer between 0 and 100.
+	 */
+	public int getProgress() {
+		return percent;
+	}
+
+	public static void main(String[] args) throws Exception {
+		VoiceImportComponent vic = new HnmTimelineMaker();
+		DatabaseLayout db = new DatabaseLayout(vic);
+		vic.compute();
+	}
 }

marytts-builder/src/main/java/marytts/tools/voiceimport/JoinCostFileMaker.java

@@ -51,286 +51,256 @@ import marytts.util.data.Datagram;
 import marytts.util.data.ESTTrackReader;
 import marytts.util.data.MaryHeader;
 
-
 public class JoinCostFileMaker extends VoiceImportComponent {
-    
-    private DatabaseLayout db = null;
-    private int percent = 0;
-    private String mcepExt = ".mcep";    
-    private int numberOfFeatures = 0;
-    private float[] fw = null;
-    private String[] wfun = null;
-    
-    public final String JOINCOSTFILE = "JoinCostFileMaker.joinCostFile";
-    public final String MCEPTIMELINE = "JoinCostFileMaker.mcepTimeline";
-    public final String UNITFILE = "JoinCostFileMaker.unitFile";
-    public final String FEATUREFILE = "JoinCostFileMaker.acfeatureFile";
-    public final String WEIGHTSFILE = "JoinCostFileMaker.weightsFile";
-    public final String MCEPDIR = "JoinCostFileMaker.mcepDir";
-    
-    public String getName(){
-        return "JoinCostFileMaker";
-    }
-    
-    @Override
-    protected void initialiseComp()
-    {
-        //make sure that we have a weights file
-        File weightsFile = new File(getProp(WEIGHTSFILE));
-        if (!weightsFile.exists()){
-            try{
-                PrintWriter weightsOut =
-                    new PrintWriter(
-                            new OutputStreamWriter(
-                                    new FileOutputStream(weightsFile),"UTF-8"));
-                printWeightsFile(weightsOut);
-            } catch (Exception e){
-                System.out.println("Warning: no join cost weights file "
-                        +getProp(WEIGHTSFILE)
-                        +"; JoinCostFileMaker will not run.");
-            }            
-        }
-    }
-    
-    public SortedMap<String,String> getDefaultProps(DatabaseLayout db){
-        this.db = db;
-       if (props == null){
-           props = new TreeMap<String,String>();
-           String filedir = db.getProp(db.FILEDIR);
-           props.put(JOINCOSTFILE, filedir + "joinCostFeatures"+db.getProp(db.MARYEXT));
-           props.put(MCEPTIMELINE, filedir + "timeline_mcep"+db.getProp(db.MARYEXT));
-           props.put(UNITFILE, filedir + "halfphoneUnits"+db.getProp(db.MARYEXT));
-           props.put(FEATUREFILE, filedir + "halfphoneFeatures_ac"+db.getProp(db.MARYEXT));
-           props.put(WEIGHTSFILE, db.getProp(db.CONFIGDIR) + "joinCostWeights.txt");
-           props.put(MCEPDIR, db.getProp(db.ROOTDIR) + "mcep" + System.getProperty("file.separator"));
-       }
-       return props;
-    }
-    
-    protected void setupHelp(){         
-        props2Help = new TreeMap<String,String>();
-        props2Help.put(JOINCOSTFILE, "file containing all halfphone units and their join cost features."
-                +" Will be created by this module");
-        props2Help.put(MCEPTIMELINE, "file containing all mcep files");
-        props2Help.put(UNITFILE, "file containing all halfphone units");
-        props2Help.put(FEATUREFILE, "file containing all halfphone features including acoustic features");
-        props2Help.put(WEIGHTSFILE, "file containing the list of join cost weights and their weights");
-        props2Help.put(MCEPDIR, "directory containing the mcep files");
-        
-    }
-    
-    @Override
-    public boolean compute() throws IOException, MaryConfigurationException
-    {
-        System.out.print("---- Making the join cost file\n");
-        
-        /* Export the basename list into an array of strings */
-        String[] baseNameArray = bnl.getListAsArray();
-        
-        /* Read the number of mel cepstra from the first melcep file */
-        ESTTrackReader firstMcepFile = new ESTTrackReader(getProp(MCEPDIR) 
-                + baseNameArray[0] + mcepExt);
-        int numberOfMelcep = firstMcepFile.getNumChannels();
-        firstMcepFile = null; // Free the memory taken by the file
-        
-        /* Make a new join cost file to write to */
-        DataOutputStream jcf = new DataOutputStream( new BufferedOutputStream( new FileOutputStream(getProp(JOINCOSTFILE)) ) );
-        
-        /**********/
-        /* HEADER */
-        /**********/
-        /* Make a new mary header and ouput it */
-        MaryHeader hdr = new MaryHeader( MaryHeader.JOINFEATS );
-        hdr.writeTo( jcf );
-        hdr = null;
-        
-        /****************************/
-        /* WEIGHTING FUNCTION SPECS */
-        /****************************/
-        /* Load the weight vectors */
-        Object[] weightData = JoinCostFeatures.readJoinCostWeightsFile(getProp(WEIGHTSFILE));
-        fw = (float[]) weightData[0];
-        wfun = (String[]) weightData[1];
-        numberOfFeatures = fw.length;
-        int numberOfProsodyFeatures = 2; 
-        
-        /* Output those vectors */
-        jcf.writeInt( fw.length );
-        for ( int i = 0; i < fw.length; i++ ) {
-            jcf.writeFloat( fw[i] );
-            jcf.writeUTF( wfun[i] );
-        }
-        /* Clean the house */
-        fw = null;
-        wfun = null;
-        
-        /************/
-        /* FEATURES */
-        /************/
-        
-        /* Open the melcep timeline */
-        TimelineReader mcep = new TimelineReader(getProp(MCEPTIMELINE));
-        
-        /* Open the unit file */
-        UnitFileReader ufr = new UnitFileReader(getProp(UNITFILE));
-        
-        // And the feature file
-        FeatureFileReader features = new FeatureFileReader(getProp(FEATUREFILE));
-        
-        /* Start writing the features: */
 
-        /* - write the number of features: */
-        jcf.writeInt( ufr.getNumberOfUnits() );
-        int unitSampleFreq = ufr.getSampleRate();
-        if (unitSampleFreq != mcep.getSampleRate()) {
-            throw new MaryConfigurationException("Cannot currently deal with different sample rates in unit and mcep files.");
-        }
-        long unitPosition = 0l;
-        int unitDuration = 0;
-        
-        /* Check the consistency between the number of join cost features
-         * and the number of Mel-cepstrum coefficients */
-        Datagram dat = mcep.getDatagram( 0 );
-        if ( dat.getData().length != (4*(numberOfFeatures-numberOfProsodyFeatures)) ) {
-            throw new MaryConfigurationException( "The number of join cost features [" + numberOfFeatures
-                    + "] read from the join cost weight config file [" + getProp(WEIGHTSFILE)
-                    + "] does not match the number of Mel Cepstra [" + (dat.getData().length / 4)
-                    + "] found in the Mel-Cepstrum timeline file [" + getProp(MCEPTIMELINE)
-                    + "], plus ["+numberOfProsodyFeatures+"] for the prosody features." );
-        }
-        
-        /* Loop through the units */
-        int fiLogF0 = features.getFeatureDefinition().getFeatureIndex("unit_logf0");
-        int fiLogF0Delta = features.getFeatureDefinition().getFeatureIndex("unit_logf0delta");
-        for ( int i = 0; i < ufr.getNumberOfUnits(); i++ ) {
-            percent = 100*i/ufr.getNumberOfUnits();
+	private DatabaseLayout db = null;
+	private int percent = 0;
+	private String mcepExt = ".mcep";
+	private int numberOfFeatures = 0;
+	private float[] fw = null;
+	private String[] wfun = null;
 
-            FeatureVector fv = features.getFeatureVector(i);
-            float logf0 = fv.getContinuousFeature(fiLogF0);
-            float logf0Delta = fv.getContinuousFeature(fiLogF0Delta);
-            
-            // logf0 is the value in the middle, i.e.
-            // leftF0 + 0.5 * logf0Delta == logf0, and
-            // logf0 + 0.5 * logf0Delta == rightF0
-            float leftLogF0 = logf0 - 0.5f * logf0Delta;
-            float rightLogF0 = logf0 + 0.5f * logf0Delta;
-            
-            /* Read the unit */
-            unitPosition = ufr.getUnit(i).startTime;
-            unitDuration = ufr.getUnit(i).duration;
-            
-            /* If the unit is not a START or END marker
-             * and has length > 0: */
-            if ( unitDuration != -1 && unitDuration > 0 ) {
-                
-                Datagram leftMCEP = mcep.getDatagram(unitPosition);
-                jcf.write(leftMCEP.getData());
-                jcf.writeFloat(leftLogF0);
-                jcf.writeFloat(logf0Delta);
-                
-                /* -- COMPUTE the RIGHT JCFs: */
-                Datagram rightMCEP;
-                boolean useRightContext = false;
+	public final String JOINCOSTFILE = "JoinCostFileMaker.joinCostFile";
+	public final String MCEPTIMELINE = "JoinCostFileMaker.mcepTimeline";
+	public final String UNITFILE = "JoinCostFileMaker.unitFile";
+	public final String FEATUREFILE = "JoinCostFileMaker.acfeatureFile";
+	public final String WEIGHTSFILE = "JoinCostFileMaker.weightsFile";
+	public final String MCEPDIR = "JoinCostFileMaker.mcepDir";
 
-                if (useRightContext) {
-                    // the right context datagram:
-                    rightMCEP = mcep.getDatagram(unitPosition+unitDuration);
-                } else {
-                    /* Crawl along the datagrams until we trespass the end of the unit: */
-                    long endPoint = unitPosition;
-                    dat = mcep.getDatagram(endPoint);
-                    while (endPoint + dat.getDuration() < unitPosition+unitDuration) {
-                        endPoint += dat.getDuration();
-                        dat = mcep.getDatagram(endPoint);
-                    }
-                    rightMCEP = dat;
-                }
-                jcf.write(rightMCEP.getData());
-                jcf.writeFloat(rightLogF0);
-                jcf.writeFloat(logf0Delta);
-            }
-            
-            /* If the unit is a START or END marker, output NaN for all features */
-            else {
-                for (int j=0; j<2*numberOfFeatures; j++) {
-                    jcf.writeFloat(Float.NaN);
-                }
-            }                
-        }
-        
-        jcf.close();
-        System.out.println("---- Join Cost file done.\n\n");
-        System.out.println("Number of processed units: " + ufr.getNumberOfUnits() );
-        
-        JoinCostFeatures tester = new JoinCostFeatures(getProp(JOINCOSTFILE));
-        int unitsOnDisk = tester.getNumberOfUnits();
-        if (unitsOnDisk == ufr.getNumberOfUnits()) {
-            System.out.println("Can read right number of units");
-            return true;
-        } else {
-            System.out.println("Read wrong number of units: "+unitsOnDisk);
-            return false;
-        }
-    }
-    
-    /**
-     * Provide the progress of computation, in percent, or -1 if
-     * that feature is not implemented.
-     * @return -1 if not implemented, or an integer between 0 and 100.
-     */
-    public int getProgress()
-    {
-        return percent;
-    }
+	public String getName() {
+		return "JoinCostFileMaker";
+	}
 
-    private void printWeightsFile(PrintWriter weightsOut)throws Exception{
-        weightsOut.println("# This file lists the weights and weighting functions to be used for\n"
-                +"# creating the MARY join cost file, joinCostFeature.mry .\n"
-                +"#\n"
-                +"# Lines starting with '#' are ignored; they can be used for comments\n"
-                +"# anywhere in the file. Empty lines are also ignored.\n"
-                +"# Entries must have the following form:\n"
-                +"# \n"
-                +"# <feature index> : <weight value> <weighting function> <optional weighting function parameter>\n"
-                +"# \n"
-                +"# The <feature index> is an integer value from 0 to the number of join cost features\n"
-                +"# minus one. It is used for readability, but is ignored when parsing the file.\n"
-                +"# The database import process will nevertheless check that the number of valid\n"
-                +"# lines corresponds to the number of join cost features specified from external\n"
-                +"# constraints (such as the order of the Mel-Cepstra).\n"
-                +"#\n"
-                +"# The <weight value> is a float value in text format.\n"
-                +"#\n"
-                +"# The <weighting function> is a string, for the moment one of \"linear\" or \"step\".\n"
-                +"#\n"
-                +"# The <optional weighting function parameter> is a string giving additional optional\n"
-                +"# info about the weighting function:\n"
-                +"# - \"linear\" does not take an optional argument;\n"
-                +"# - \"step\" takes a threshold position argument, e.g. \"step 20%\" means a step function\n"
-                +"#   with weighs 0 when the join feature difference is less than 20%, and applies\n"
-                +"#   the weight value when the join feature difference is 20% or more.\n"
-                +"#\n"
-                +"# THIS FILE WAS GENERATED AUTOMATICALLY\n"
-                +"\n"
-                +"# Weights applied to the Mel-cepstra:\n"
-                +"0  : 1.0 linear\n"
-                +"1  : 1.0 linear\n"
-                +"2  : 1.0 linear\n"
-                +"3  : 1.0 linear\n"
-                +"4  : 1.0 linear\n"
-                +"5  : 1.0 linear\n"
-                +"6  : 1.0 linear\n"
-                +"7  : 1.0 linear\n"
-                +"8  : 1.0 linear\n"
-                +"9  : 1.0 linear\n"
-                +"10 : 1.0 linear\n"
-                +"11 : 1.0 linear\n"
-                +"\n"
-                +"# Weight applied to the log F0 and log F0 delta parameters:\n"
-                +"12 : 10.0 linear\n"
-                +"13 : 0.5 linear");
-        weightsOut.flush();
-        weightsOut.close();
-    }
+	@Override
+	protected void initialiseComp() {
+		// make sure that we have a weights file
+		File weightsFile = new File(getProp(WEIGHTSFILE));
+		if (!weightsFile.exists()) {
+			try {
+				PrintWriter weightsOut = new PrintWriter(new OutputStreamWriter(new FileOutputStream(weightsFile), "UTF-8"));
+				printWeightsFile(weightsOut);
+			} catch (Exception e) {
+				System.out.println("Warning: no join cost weights file " + getProp(WEIGHTSFILE)
+						+ "; JoinCostFileMaker will not run.");
+			}
+		}
+	}
+
+	public SortedMap<String, String> getDefaultProps(DatabaseLayout db) {
+		this.db = db;
+		if (props == null) {
+			props = new TreeMap<String, String>();
+			String filedir = db.getProp(db.FILEDIR);
+			props.put(JOINCOSTFILE, filedir + "joinCostFeatures" + db.getProp(db.MARYEXT));
+			props.put(MCEPTIMELINE, filedir + "timeline_mcep" + db.getProp(db.MARYEXT));
+			props.put(UNITFILE, filedir + "halfphoneUnits" + db.getProp(db.MARYEXT));
+			props.put(FEATUREFILE, filedir + "halfphoneFeatures_ac" + db.getProp(db.MARYEXT));
+			props.put(WEIGHTSFILE, db.getProp(db.CONFIGDIR) + "joinCostWeights.txt");
+			props.put(MCEPDIR, db.getProp(db.ROOTDIR) + "mcep" + System.getProperty("file.separator"));
+		}
+		return props;
+	}
+
+	protected void setupHelp() {
+		props2Help = new TreeMap<String, String>();
+		props2Help.put(JOINCOSTFILE, "file containing all halfphone units and their join cost features."
+				+ " Will be created by this module");
+		props2Help.put(MCEPTIMELINE, "file containing all mcep files");
+		props2Help.put(UNITFILE, "file containing all halfphone units");
+		props2Help.put(FEATUREFILE, "file containing all halfphone features including acoustic features");
+		props2Help.put(WEIGHTSFILE, "file containing the list of join cost weights and their weights");
+		props2Help.put(MCEPDIR, "directory containing the mcep files");
+
+	}
+
+	@Override
+	public boolean compute() throws IOException, MaryConfigurationException {
+		System.out.print("---- Making the join cost file\n");
+
+		/* Export the basename list into an array of strings */
+		String[] baseNameArray = bnl.getListAsArray();
+
+		/* Read the number of mel cepstra from the first melcep file */
+		ESTTrackReader firstMcepFile = new ESTTrackReader(getProp(MCEPDIR) + baseNameArray[0] + mcepExt);
+		int numberOfMelcep = firstMcepFile.getNumChannels();
+		firstMcepFile = null; // Free the memory taken by the file
+
+		/* Make a new join cost file to write to */
+		DataOutputStream jcf = new DataOutputStream(new BufferedOutputStream(new FileOutputStream(getProp(JOINCOSTFILE))));
+
+		/**********/
+		/* HEADER */
+		/**********/
+		/* Make a new mary header and ouput it */
+		MaryHeader hdr = new MaryHeader(MaryHeader.JOINFEATS);
+		hdr.writeTo(jcf);
+		hdr = null;
+
+		/****************************/
+		/* WEIGHTING FUNCTION SPECS */
+		/****************************/
+		/* Load the weight vectors */
+		Object[] weightData = JoinCostFeatures.readJoinCostWeightsFile(getProp(WEIGHTSFILE));
+		fw = (float[]) weightData[0];
+		wfun = (String[]) weightData[1];
+		numberOfFeatures = fw.length;
+		int numberOfProsodyFeatures = 2;
+
+		/* Output those vectors */
+		jcf.writeInt(fw.length);
+		for (int i = 0; i < fw.length; i++) {
+			jcf.writeFloat(fw[i]);
+			jcf.writeUTF(wfun[i]);
+		}
+		/* Clean the house */
+		fw = null;
+		wfun = null;
+
+		/************/
+		/* FEATURES */
+		/************/
+
+		/* Open the melcep timeline */
+		TimelineReader mcep = new TimelineReader(getProp(MCEPTIMELINE));
+
+		/* Open the unit file */
+		UnitFileReader ufr = new UnitFileReader(getProp(UNITFILE));
+
+		// And the feature file
+		FeatureFileReader features = new FeatureFileReader(getProp(FEATUREFILE));
+
+		/* Start writing the features: */
+
+		/* - write the number of features: */
+		jcf.writeInt(ufr.getNumberOfUnits());
+		int unitSampleFreq = ufr.getSampleRate();
+		if (unitSampleFreq != mcep.getSampleRate()) {
+			throw new MaryConfigurationException("Cannot currently deal with different sample rates in unit and mcep files.");
+		}
+		long unitPosition = 0l;
+		int unitDuration = 0;
+
+		/*
+		 * Check the consistency between the number of join cost features and the number of Mel-cepstrum coefficients
+		 */
+		Datagram dat = mcep.getDatagram(0);
+		if (dat.getData().length != (4 * (numberOfFeatures - numberOfProsodyFeatures))) {
+			throw new MaryConfigurationException("The number of join cost features [" + numberOfFeatures
+					+ "] read from the join cost weight config file [" + getProp(WEIGHTSFILE)
+					+ "] does not match the number of Mel Cepstra [" + (dat.getData().length / 4)
+					+ "] found in the Mel-Cepstrum timeline file [" + getProp(MCEPTIMELINE) + "], plus ["
+					+ numberOfProsodyFeatures + "] for the prosody features.");
+		}
+
+		/* Loop through the units */
+		int fiLogF0 = features.getFeatureDefinition().getFeatureIndex("unit_logf0");
+		int fiLogF0Delta = features.getFeatureDefinition().getFeatureIndex("unit_logf0delta");
+		for (int i = 0; i < ufr.getNumberOfUnits(); i++) {
+			percent = 100 * i / ufr.getNumberOfUnits();
+
+			FeatureVector fv = features.getFeatureVector(i);
+			float logf0 = fv.getContinuousFeature(fiLogF0);
+			float logf0Delta = fv.getContinuousFeature(fiLogF0Delta);
+
+			// logf0 is the value in the middle, i.e.
+			// leftF0 + 0.5 * logf0Delta == logf0, and
+			// logf0 + 0.5 * logf0Delta == rightF0
+			float leftLogF0 = logf0 - 0.5f * logf0Delta;
+			float rightLogF0 = logf0 + 0.5f * logf0Delta;
+
+			/* Read the unit */
+			unitPosition = ufr.getUnit(i).startTime;
+			unitDuration = ufr.getUnit(i).duration;
+
+			/*
+			 * If the unit is not a START or END marker and has length > 0:
+			 */
+			if (unitDuration != -1 && unitDuration > 0) {
+
+				Datagram leftMCEP = mcep.getDatagram(unitPosition);
+				jcf.write(leftMCEP.getData());
+				jcf.writeFloat(leftLogF0);
+				jcf.writeFloat(logf0Delta);
+
+				/* -- COMPUTE the RIGHT JCFs: */
+				Datagram rightMCEP;
+				boolean useRightContext = false;
+
+				if (useRightContext) {
+					// the right context datagram:
+					rightMCEP = mcep.getDatagram(unitPosition + unitDuration);
+				} else {
+					/* Crawl along the datagrams until we trespass the end of the unit: */
+					long endPoint = unitPosition;
+					dat = mcep.getDatagram(endPoint);
+					while (endPoint + dat.getDuration() < unitPosition + unitDuration) {
+						endPoint += dat.getDuration();
+						dat = mcep.getDatagram(endPoint);
+					}
+					rightMCEP = dat;
+				}
+				jcf.write(rightMCEP.getData());
+				jcf.writeFloat(rightLogF0);
+				jcf.writeFloat(logf0Delta);
+			}
+
+			/* If the unit is a START or END marker, output NaN for all features */
+			else {
+				for (int j = 0; j < 2 * numberOfFeatures; j++) {
+					jcf.writeFloat(Float.NaN);
+				}
+			}
+		}
+
+		jcf.close();
+		System.out.println("---- Join Cost file done.\n\n");
+		System.out.println("Number of processed units: " + ufr.getNumberOfUnits());
+
+		JoinCostFeatures tester = new JoinCostFeatures(getProp(JOINCOSTFILE));
+		int unitsOnDisk = tester.getNumberOfUnits();
+		if (unitsOnDisk == ufr.getNumberOfUnits()) {
+			System.out.println("Can read right number of units");
+			return true;
+		} else {
+			System.out.println("Read wrong number of units: " + unitsOnDisk);
+			return false;
+		}
+	}
+
+	/**
+	 * Provide the progress of computation, in percent, or -1 if that feature is not implemented.
+	 * 
+	 * @return -1 if not implemented, or an integer between 0 and 100.
+	 */
+	public int getProgress() {
+		return percent;
+	}
+
+	private void printWeightsFile(PrintWriter weightsOut) throws Exception {
+		weightsOut.println("# This file lists the weights and weighting functions to be used for\n"
+				+ "# creating the MARY join cost file, joinCostFeature.mry .\n" + "#\n"
+				+ "# Lines starting with '#' are ignored; they can be used for comments\n"
+				+ "# anywhere in the file. Empty lines are also ignored.\n" + "# Entries must have the following form:\n"
+				+ "# \n" + "# <feature index> : <weight value> <weighting function> <optional weighting function parameter>\n"
+				+ "# \n" + "# The <feature index> is an integer value from 0 to the number of join cost features\n"
+				+ "# minus one. It is used for readability, but is ignored when parsing the file.\n"
+				+ "# The database import process will nevertheless check that the number of valid\n"
+				+ "# lines corresponds to the number of join cost features specified from external\n"
+				+ "# constraints (such as the order of the Mel-Cepstra).\n" + "#\n"
+				+ "# The <weight value> is a float value in text format.\n" + "#\n"
+				+ "# The <weighting function> is a string, for the moment one of \"linear\" or \"step\".\n" + "#\n"
+				+ "# The <optional weighting function parameter> is a string giving additional optional\n"
+				+ "# info about the weighting function:\n" + "# - \"linear\" does not take an optional argument;\n"
+				+ "# - \"step\" takes a threshold position argument, e.g. \"step 20%\" means a step function\n"
+				+ "#   with weighs 0 when the join feature difference is less than 20%, and applies\n"
+				+ "#   the weight value when the join feature difference is 20% or more.\n" + "#\n"
+				+ "# THIS FILE WAS GENERATED AUTOMATICALLY\n" + "\n" + "# Weights applied to the Mel-cepstra:\n"
+				+ "0  : 1.0 linear\n" + "1  : 1.0 linear\n" + "2  : 1.0 linear\n" + "3  : 1.0 linear\n" + "4  : 1.0 linear\n"
+				+ "5  : 1.0 linear\n" + "6  : 1.0 linear\n" + "7  : 1.0 linear\n" + "8  : 1.0 linear\n" + "9  : 1.0 linear\n"
+				+ "10 : 1.0 linear\n" + "11 : 1.0 linear\n" + "\n"
+				+ "# Weight applied to the log F0 and log F0 delta parameters:\n" + "12 : 10.0 linear\n" + "13 : 0.5 linear");
+		weightsOut.flush();
+		weightsOut.close();
+	}
 
 }

marytts-builder/src/main/java/marytts/tools/voiceimport/JoinCostPrecomputer.java

@@ -51,185 +51,183 @@ import marytts.unitselection.select.JoinCostFeatures;
 import marytts.unitselection.select.PrecompiledJoinCostReader;
 import marytts.util.data.MaryHeader;
 
+public class JoinCostPrecomputer extends VoiceImportComponent {
 
-public class JoinCostPrecomputer extends VoiceImportComponent
-{
-    
-    private DatabaseLayout db = null;
-    private int percent = 0;
-    
-    private int numberOfFeatures = 0;
-    private float[] fw = null;
-    private String[] wfun = null;
-    
-    public final String JOINCOSTFILE = "JoinCostPrecomputer.joinCostFile";
-    public final String JOINCOSTFEATURESFILE = "JoinCostPrecomputer.joinCostFeaturesFile";
-    public final String UNITFEATURESFILE = "JoinCostPrecomputer.unitFeaturesFile";
-    public final String UNITFILE = "JoinCostPrecomputer.unitFile";
-    
-    public String getName(){
-        return "JoinCostPrecomputer";
-    }
- 
-    public SortedMap getDefaultProps(DatabaseLayout db){
-        this.db = db;
-       if (props == null){
-           props = new TreeMap();
-           String filedir = db.getProp(db.FILEDIR);
-           String maryExt = db.getProp(db.MARYEXT);
-           props.put(JOINCOSTFILE,filedir+"joinCosts"+maryExt);
-           props.put(JOINCOSTFEATURESFILE,filedir+"joinCostFeatures"+maryExt);
-           props.put(UNITFEATURESFILE,filedir+"halfphoneFeatures"+maryExt);
-           props.put(UNITFILE,filedir+"halfphoneUnits"+maryExt);
-       }
-       return props;
-    }
-    
-    protected void setupHelp(){         
-        props2Help = new TreeMap();
-        props2Help.put(JOINCOSTFILE,"file containing the join costs for every halfphone unit pair."
-                +" Will be created by this module");
-        props2Help.put(JOINCOSTFEATURESFILE,"file containing all halfphone units and their join cost features");
-        props2Help.put(UNITFEATURESFILE,"file containing all halfphone units and their target cost features");
-        props2Help.put(UNITFILE,"file containing all halfphone units");
-    }
-    
-    @Override
-    public boolean compute() throws IOException, MaryConfigurationException
-    {
-        System.out.println("---- Precomputing join costs");
-        int retainPercent = Integer.getInteger("joincostprecomputer.retainpercent", 10).intValue();
-        int retainMin = Integer.getInteger("joincostprecomputer.retainmin", 20).intValue();
-        System.out.println("Will retain the top "+retainPercent+"% (but at least "+retainMin+") of all joins within a phone");
-        
-        /* Make a new join cost file to write to */
-        DataOutputStream jc = new DataOutputStream( new BufferedOutputStream( new FileOutputStream(getProp(JOINCOSTFILE))));
-        
-        /**********/
-        /* HEADER */
-        /**********/
-        /* Make a new mary header and ouput it */
-        MaryHeader hdr = new MaryHeader( MaryHeader.PRECOMPUTED_JOINCOSTS );
-        hdr.writeTo( jc );
-        hdr = null;
-        
-        FeatureFileReader unitFeatures = FeatureFileReader.getFeatureFileReader(getProp(UNITFEATURESFILE));
-        JoinCostFeatures joinFeatures = new JoinCostFeatures(getProp(JOINCOSTFEATURESFILE));
-        UnitFileReader units = new UnitFileReader(getProp(UNITFILE));
-        if (unitFeatures.getNumberOfUnits() != joinFeatures.getNumberOfUnits())
-            throw new IllegalStateException("Number of units in unit and join feature files does not match!");
-        if (unitFeatures.getNumberOfUnits() != units.getNumberOfUnits())
-            throw new IllegalStateException("Number of units in unit file and unit feature file does not match!");
-        int numUnits = unitFeatures.getNumberOfUnits();
-        FeatureDefinition def = unitFeatures.getFeatureDefinition();
-        int iPhoneme = def.getFeatureIndex("phone");
-        int nPhonemes = def.getNumberOfValues(iPhoneme);
-        List[] left = new List[nPhonemes]; // left half phones grouped by phone
-        List[] right = new List[nPhonemes]; // right half phones grouped by phone
-        for (int i=0; i<nPhonemes; i++) {
-            left[i] = new ArrayList();
-            right[i] = new ArrayList();
-        }
-        int iLeftRight = def.getFeatureIndex("halfphone_lr");
-        byte vLeft = def.getFeatureValueAsByte(iLeftRight, "L");
-        byte vRight = def.getFeatureValueAsByte(iLeftRight, "R");
-        for (int i=0; i<numUnits; i++) {
-            FeatureVector fv = unitFeatures.getFeatureVector(i); 
-            int phone = fv.getFeatureAsInt(iPhoneme);
-            assert 0 <= phone && phone < nPhonemes;
-            byte lr = fv.getByteFeature(iLeftRight);
-            Unit u = units.getUnit(i);
-            if (lr == vLeft) {
-                left[phone].add(u);
-            } else if (lr == vRight) {
-                right[phone].add(u);
-            }
-        }
-        
-        System.out.println("Sorted units by phone and halfphone. Now computing costs.");
-        int totalLeftUnits = 0;
-        for (int i=0; i<nPhonemes; i++) {
-            totalLeftUnits += left[i].size();
-        }
-        jc.writeInt(totalLeftUnits);
-        for (int i=0; i<nPhonemes; i++) {
-            String phoneSymbol = def.getFeatureValueAsString(iPhoneme, i);
-            int nLeftPhoneme = left[i].size();
-            int nRightPhoneme = right[i].size();
-            System.out.println(phoneSymbol+": "+nLeftPhoneme+" left, "+nRightPhoneme+" right half phones");
-            for (int j=0; j<nLeftPhoneme; j++) {
-                Unit uleft = (Unit) left[i].get(j);
-                SortedMap sortedCosts = new TreeMap();
-                int ileft = uleft.index;
-                //System.out.println("Left unit "+j+" (index "+ileft+")");
-                jc.writeInt(ileft);
-                // Now for this left halfphone, compute the cost of joining to each
-                // right halfphones of the same phone, and remember only the best.
-                for (int k=0; k<nRightPhoneme; k++) {
-                    Unit uright = (Unit) right[i].get(k);
-                    int iright = uright.index;
-                    //System.out.println("right unit "+k+" (index "+iright+")");
-                    double cost = joinFeatures.cost(ileft, iright);
-                    Double dCost = new Double(cost);
-                    // make sure we don't overwrite any existing entry:
-                    if (!sortedCosts.containsKey(dCost)) {
-                        sortedCosts.put(dCost, uright);
-                    } else {
-                        Object value = sortedCosts.get(dCost);
-                        List newVal = new ArrayList();
-                        if (value instanceof List)
-                            newVal.addAll((List)value);
-                        else
-                            newVal.add(value);
-                        newVal.add(uright);
-                        sortedCosts.put(dCost, newVal);
-                    }
-                }
-                // Number of joins we will retain:
-                int nRetain = nRightPhoneme * retainPercent / 100;
-                if (nRetain < retainMin) nRetain = retainMin;
-                if (nRetain > nRightPhoneme) nRetain = nRightPhoneme;
-                jc.writeInt(nRetain);
-                Iterator it=sortedCosts.keySet().iterator();
-                for (int k=0; k<nRetain; ) {
-                    Double cost = (Double) it.next();
-                    float fcost = cost.floatValue();
-                    Object ob = sortedCosts.get(cost);
-                    if (ob instanceof Unit) {
-                        Unit u = (Unit) ob;
-                        int iright = u.index;
-                        jc.writeInt(iright);
-                        jc.writeFloat(fcost);
-                        k++;
-                    } else {
-                        assert ob instanceof List;
-                        List l = (List) ob;
-                        for (Iterator li = l.iterator(); k<nRetain && li.hasNext(); ) {
-                            Unit u = (Unit) li.next();
-                            int iright = u.index;
-                            jc.writeInt(iright);
-                            jc.writeFloat(fcost);
-                            k++;
-                        }
-                    }
-                }
-            }
-            percent += 100*nLeftPhoneme/totalLeftUnits;
-        }
-        jc.close();
-        PrecompiledJoinCostReader tester = new PrecompiledJoinCostReader(getProp(JOINCOSTFILE));
-        return true;
-    }
-    
-    /**
-     * Provide the progress of computation, in percent, or -1 if
-     * that feature is not implemented.
-     * @return -1 if not implemented, or an integer between 0 and 100.
-     */
-    public int getProgress()
-    {
-        return percent;
-    }
+	private DatabaseLayout db = null;
+	private int percent = 0;
 
+	private int numberOfFeatures = 0;
+	private float[] fw = null;
+	private String[] wfun = null;
+
+	public final String JOINCOSTFILE = "JoinCostPrecomputer.joinCostFile";
+	public final String JOINCOSTFEATURESFILE = "JoinCostPrecomputer.joinCostFeaturesFile";
+	public final String UNITFEATURESFILE = "JoinCostPrecomputer.unitFeaturesFile";
+	public final String UNITFILE = "JoinCostPrecomputer.unitFile";
+
+	public String getName() {
+		return "JoinCostPrecomputer";
+	}
+
+	public SortedMap getDefaultProps(DatabaseLayout db) {
+		this.db = db;
+		if (props == null) {
+			props = new TreeMap();
+			String filedir = db.getProp(db.FILEDIR);
+			String maryExt = db.getProp(db.MARYEXT);
+			props.put(JOINCOSTFILE, filedir + "joinCosts" + maryExt);
+			props.put(JOINCOSTFEATURESFILE, filedir + "joinCostFeatures" + maryExt);
+			props.put(UNITFEATURESFILE, filedir + "halfphoneFeatures" + maryExt);
+			props.put(UNITFILE, filedir + "halfphoneUnits" + maryExt);
+		}
+		return props;
+	}
+
+	protected void setupHelp() {
+		props2Help = new TreeMap();
+		props2Help.put(JOINCOSTFILE, "file containing the join costs for every halfphone unit pair."
+				+ " Will be created by this module");
+		props2Help.put(JOINCOSTFEATURESFILE, "file containing all halfphone units and their join cost features");
+		props2Help.put(UNITFEATURESFILE, "file containing all halfphone units and their target cost features");
+		props2Help.put(UNITFILE, "file containing all halfphone units");
+	}
+
+	@Override
+	public boolean compute() throws IOException, MaryConfigurationException {
+		System.out.println("---- Precomputing join costs");
+		int retainPercent = Integer.getInteger("joincostprecomputer.retainpercent", 10).intValue();
+		int retainMin = Integer.getInteger("joincostprecomputer.retainmin", 20).intValue();
+		System.out.println("Will retain the top " + retainPercent + "% (but at least " + retainMin
+				+ ") of all joins within a phone");
+
+		/* Make a new join cost file to write to */
+		DataOutputStream jc = new DataOutputStream(new BufferedOutputStream(new FileOutputStream(getProp(JOINCOSTFILE))));
+
+		/**********/
+		/* HEADER */
+		/**********/
+		/* Make a new mary header and ouput it */
+		MaryHeader hdr = new MaryHeader(MaryHeader.PRECOMPUTED_JOINCOSTS);
+		hdr.writeTo(jc);
+		hdr = null;
+
+		FeatureFileReader unitFeatures = FeatureFileReader.getFeatureFileReader(getProp(UNITFEATURESFILE));
+		JoinCostFeatures joinFeatures = new JoinCostFeatures(getProp(JOINCOSTFEATURESFILE));
+		UnitFileReader units = new UnitFileReader(getProp(UNITFILE));
+		if (unitFeatures.getNumberOfUnits() != joinFeatures.getNumberOfUnits())
+			throw new IllegalStateException("Number of units in unit and join feature files does not match!");
+		if (unitFeatures.getNumberOfUnits() != units.getNumberOfUnits())
+			throw new IllegalStateException("Number of units in unit file and unit feature file does not match!");
+		int numUnits = unitFeatures.getNumberOfUnits();
+		FeatureDefinition def = unitFeatures.getFeatureDefinition();
+		int iPhoneme = def.getFeatureIndex("phone");
+		int nPhonemes = def.getNumberOfValues(iPhoneme);
+		List[] left = new List[nPhonemes]; // left half phones grouped by phone
+		List[] right = new List[nPhonemes]; // right half phones grouped by phone
+		for (int i = 0; i < nPhonemes; i++) {
+			left[i] = new ArrayList();
+			right[i] = new ArrayList();
+		}
+		int iLeftRight = def.getFeatureIndex("halfphone_lr");
+		byte vLeft = def.getFeatureValueAsByte(iLeftRight, "L");
+		byte vRight = def.getFeatureValueAsByte(iLeftRight, "R");
+		for (int i = 0; i < numUnits; i++) {
+			FeatureVector fv = unitFeatures.getFeatureVector(i);
+			int phone = fv.getFeatureAsInt(iPhoneme);
+			assert 0 <= phone && phone < nPhonemes;
+			byte lr = fv.getByteFeature(iLeftRight);
+			Unit u = units.getUnit(i);
+			if (lr == vLeft) {
+				left[phone].add(u);
+			} else if (lr == vRight) {
+				right[phone].add(u);
+			}
+		}
+
+		System.out.println("Sorted units by phone and halfphone. Now computing costs.");
+		int totalLeftUnits = 0;
+		for (int i = 0; i < nPhonemes; i++) {
+			totalLeftUnits += left[i].size();
+		}
+		jc.writeInt(totalLeftUnits);
+		for (int i = 0; i < nPhonemes; i++) {
+			String phoneSymbol = def.getFeatureValueAsString(iPhoneme, i);
+			int nLeftPhoneme = left[i].size();
+			int nRightPhoneme = right[i].size();
+			System.out.println(phoneSymbol + ": " + nLeftPhoneme + " left, " + nRightPhoneme + " right half phones");
+			for (int j = 0; j < nLeftPhoneme; j++) {
+				Unit uleft = (Unit) left[i].get(j);
+				SortedMap sortedCosts = new TreeMap();
+				int ileft = uleft.index;
+				// System.out.println("Left unit "+j+" (index "+ileft+")");
+				jc.writeInt(ileft);
+				// Now for this left halfphone, compute the cost of joining to each
+				// right halfphones of the same phone, and remember only the best.
+				for (int k = 0; k < nRightPhoneme; k++) {
+					Unit uright = (Unit) right[i].get(k);
+					int iright = uright.index;
+					// System.out.println("right unit "+k+" (index "+iright+")");
+					double cost = joinFeatures.cost(ileft, iright);
+					Double dCost = new Double(cost);
+					// make sure we don't overwrite any existing entry:
+					if (!sortedCosts.containsKey(dCost)) {
+						sortedCosts.put(dCost, uright);
+					} else {
+						Object value = sortedCosts.get(dCost);
+						List newVal = new ArrayList();
+						if (value instanceof List)
+							newVal.addAll((List) value);
+						else
+							newVal.add(value);
+						newVal.add(uright);
+						sortedCosts.put(dCost, newVal);
+					}
+				}
+				// Number of joins we will retain:
+				int nRetain = nRightPhoneme * retainPercent / 100;
+				if (nRetain < retainMin)
+					nRetain = retainMin;
+				if (nRetain > nRightPhoneme)
+					nRetain = nRightPhoneme;
+				jc.writeInt(nRetain);
+				Iterator it = sortedCosts.keySet().iterator();
+				for (int k = 0; k < nRetain;) {
+					Double cost = (Double) it.next();
+					float fcost = cost.floatValue();
+					Object ob = sortedCosts.get(cost);
+					if (ob instanceof Unit) {
+						Unit u = (Unit) ob;
+						int iright = u.index;
+						jc.writeInt(iright);
+						jc.writeFloat(fcost);
+						k++;
+					} else {
+						assert ob instanceof List;
+						List l = (List) ob;
+						for (Iterator li = l.iterator(); k < nRetain && li.hasNext();) {
+							Unit u = (Unit) li.next();
+							int iright = u.index;
+							jc.writeInt(iright);
+							jc.writeFloat(fcost);
+							k++;
+						}
+					}
+				}
+			}
+			percent += 100 * nLeftPhoneme / totalLeftUnits;
+		}
+		jc.close();
+		PrecompiledJoinCostReader tester = new PrecompiledJoinCostReader(getProp(JOINCOSTFILE));
+		return true;
+	}
+
+	/**
+	 * Provide the progress of computation, in percent, or -1 if that feature is not implemented.
+	 * 
+	 * @return -1 if not implemented, or an integer between 0 and 100.
+	 */
+	public int getProgress() {
+		return percent;
+	}
 
 }

marytts-builder/src/main/java/marytts/tools/voiceimport/JoinModeller.java

@@ -65,458 +65,451 @@ import marytts.unitselection.select.JoinCostFeatures;
 import marytts.util.io.General;
 import marytts.util.math.MathUtils;
 
+public class JoinModeller extends VoiceImportComponent {
 
-public class JoinModeller extends VoiceImportComponent
-{
-    
-    private DatabaseLayout db = null;
-    private int percent = 0;
-      
-    private HMMData htsData = null;  /* for using the function readFeatureList() */
-    private Vector<String> featureList = null;
-    private Map<String,String> feat2shortFeat = new HashMap<String, String>();
-    
-    private int numberOfFeatures = 0;
-    private float[] fw = null;
-    private String[] wfun = null;
-    FileWriter statsStream = null;
-    FileWriter mmfStream = null;
-    FileWriter fullStream = null;
+	private DatabaseLayout db = null;
+	private int percent = 0;
 
-    
-    public final String JOINCOSTFEATURESFILE = "JoinModeller.joinCostFeaturesFile";
-    public final String UNITFEATURESFILE = "JoinModeller.unitFeaturesFile";
-    public final String UNITFILE = "JoinModeller.unitFile";
-    public final String STATSFILE = "JoinModeller.statsFile";
-    public final String MMFFILE = "JoinModeller.mmfFile";
-    public final String FULLFILE = "JoinModeller.fullFile";
-    public final String CXCHEDFILE = "JoinModeller.cxcJoinFile";
-    public final String JOINTREEFILE = "JoinModeller.joinTreeFile";
-    public final String CNVHEDFILE = "JoinModeller.cnvJoinFile";
-    public final String TRNCONFFILE = "JoinModeller.trnFile";
-    public final String CNVCONFFILE = "JoinModeller.cnvFile"; 
-    public final String HHEDCOMMAND = "JoinModeller.hhedCommand";
-    public final String FEATURELISTFILE = "JoinModeller.featureListFile";
-    public final String TRICKYPHONESFILE = "JoinModeller.trickyPhonesFile";
-    
-    public JoinModeller()
-    {
-        
-    }
-    
-    public String getName(){
-        return "JoinModeller";
-    }
- 
-    public SortedMap<String,String> getDefaultProps(DatabaseLayout db){
-        this.db = db;
-       if (props == null){
-           props = new TreeMap<String,String>();
-           String filedir = db.getProp(db.FILEDIR);
-           String maryExt = db.getProp(db.MARYEXT);
-           props.put(JOINCOSTFEATURESFILE,filedir+"joinCostFeatures"+maryExt);
-           props.put(UNITFEATURESFILE,filedir+"halfphoneFeatures"+maryExt);
-           props.put(UNITFILE,filedir+"halfphoneUnits"+maryExt);
-           props.put(STATSFILE,filedir+"stats"+maryExt);
-           props.put(MMFFILE,filedir+"join_mmf"+maryExt);
-           props.put(FULLFILE, filedir+"fullList"+maryExt);
-           props.put(FEATURELISTFILE, filedir+"/mary/featureListFile.txt");
-           props.put(TRICKYPHONESFILE, filedir+"/mary/trickyPhones.txt");
-           props.put(CXCHEDFILE, filedir+"cxc_join.hed");
-           props.put(JOINTREEFILE, filedir+"join_tree.inf");
-           props.put(CNVHEDFILE, filedir+"cnv_join.hed");
-           props.put(TRNCONFFILE, filedir+"trn.cnf");
-           props.put(CNVCONFFILE, filedir+"cnv.cnf");
-           props.put(HHEDCOMMAND, "/project/mary/marcela/sw/HTS_2.0.1/htk/bin/HHEd");
-       }
-       return props;
-    }
-    
-    protected void setupHelp(){         
-        props2Help = new TreeMap<String,String>();
-        props2Help.put(JOINCOSTFEATURESFILE,"file containing all halfphone units and their join cost features");
-        props2Help.put(UNITFEATURESFILE,"file containing all halfphone units and their target cost features");
-        props2Help.put(UNITFILE,"file containing all halfphone units");      
-        props2Help.put(STATSFILE,"output file containing statistics of the models in HTK stats format");
-        props2Help.put(MMFFILE,"output file containing one state HMM models, HTK format, representing join models (mean and variances are calculated in this class)");
-        props2Help.put(FULLFILE,"output file containing the full list of HMM model names");
-        props2Help.put(FEATURELISTFILE,"feature list for making fullcontext names and questions");
-        props2Help.put(TRICKYPHONESFILE,"list of aliases for tricky phones, so HTK-HHEd command can handle them.");
-        props2Help.put(CXCHEDFILE,"HTK hed file used by HHEd, load stats file, contains questions for decision tree-based context clustering and outputs result in join-tree.inf");
-        props2Help.put(CNVHEDFILE,"HTK hed file used by HHEd to convert trees and mmf into hts_engine format");
-        props2Help.put(TRNCONFFILE,"HTK configuration file for context clustering");
-        props2Help.put(CNVCONFFILE,"HTK configuration file for converting to hts_engine format");
-        props2Help.put(HHEDCOMMAND,"HTS-HTK HHEd command, HTS version minimum HTS_2.0.1");
-    }
-    
-    public boolean compute() throws IOException, Exception
-    {
-        System.out.println("\n---- Training join models\n");       
-        
-        FeatureFileReader unitFeatures = FeatureFileReader.getFeatureFileReader(getProp(UNITFEATURESFILE));
-        JoinCostFeatures joinFeatures = new JoinCostFeatures(getProp(JOINCOSTFEATURESFILE));
-        UnitFileReader units = new UnitFileReader(getProp(UNITFILE));
-        FeatureDefinition def = unitFeatures.getFeatureDefinition();
-        
-        // for creating a contexttranslator object we need to check if there is 
-        PhoneTranslator contextTranslator = null;
-        // Check if there are tricky phones          
-        if( HMMVoiceMakeData.checkTrickyPhones(db.getAllophoneSet(), getProp(TRICKYPHONESFILE)) )        
-            contextTranslator = new PhoneTranslator(new FileInputStream(getProp(TRICKYPHONESFILE)));
-        else
-            contextTranslator = new PhoneTranslator(null); 
-        
-        
-        featureList = new Vector<String>(); 
-        readFeatureList(getProp(FEATURELISTFILE), def);
-              
-        statsStream = new FileWriter(getProp(STATSFILE));
-        mmfStream   = new FileWriter(getProp(MMFFILE));
-        fullStream  = new FileWriter(getProp(FULLFILE));
-        // output HTK model definition and dummy state-transition matrix, macro ~t "trP_1"
-        int numFeatures = joinFeatures.getNumberOfFeatures();             
-        mmfStream.write("~o\n" + "<VECSIZE> " + numFeatures 
-                + " <USER><DIAGC>\n"  
-                + "~t \"trP_1\"\n<TRANSP> 3\n" + "0 1 0\n0 0 1\n0 0 0\n");
-       
-       
-         
-        if (unitFeatures.getNumberOfUnits() != joinFeatures.getNumberOfUnits())
-            throw new IllegalStateException("Number of units in unit and join feature files does not match!");
-        if (unitFeatures.getNumberOfUnits() != units.getNumberOfUnits())
-            throw new IllegalStateException("Number of units in unit file and unit feature file does not match!");
-        int numUnits = unitFeatures.getNumberOfUnits();
-        //----FeatureDefinition def = unitFeatures.getFeatureDefinition();
+	private HMMData htsData = null; /* for using the function readFeatureList() */
+	private Vector<String> featureList = null;
+	private Map<String, String> feat2shortFeat = new HashMap<String, String>();
 
-        int iPhoneme = def.getFeatureIndex("phone");
-        int nPhonemes = def.getNumberOfValues(iPhoneme);
-        int iLeftRight = def.getFeatureIndex("halfphone_lr");
-        byte vLeft = def.getFeatureValueAsByte(iLeftRight, "L");
-        byte vRight = def.getFeatureValueAsByte(iLeftRight, "R");
-        int iEdge = def.getFeatureIndex("edge");
-        byte vNoEdge = def.getFeatureValueAsByte(iEdge, "0");
-        byte vStartEdge = def.getFeatureValueAsByte(iEdge, "start");
-        byte vEndEdge = def.getFeatureValueAsByte(iEdge, "end");
+	private int numberOfFeatures = 0;
+	private float[] fw = null;
+	private String[] wfun = null;
+	FileWriter statsStream = null;
+	FileWriter mmfStream = null;
+	FileWriter fullStream = null;
 
-        
-        Map<String,Set<double[]>> uniqueFeatureVectors = new HashMap<String, Set<double[]>>();
-        
-        // Now look at all pairs of adjacent units
-        FeatureVector fvNext = unitFeatures.getFeatureVector(0);
-        for (int i=0; i<numUnits-1; i++) {
-            percent = 100 * (i+1) / numUnits;
-            
-            FeatureVector fv = fvNext; 
-            byte edge = fv.getByteFeature(iEdge);
-            fvNext = unitFeatures.getFeatureVector(i+1);
-            byte edgeNext = fvNext.getByteFeature(iEdge);
+	public final String JOINCOSTFEATURESFILE = "JoinModeller.joinCostFeaturesFile";
+	public final String UNITFEATURESFILE = "JoinModeller.unitFeaturesFile";
+	public final String UNITFILE = "JoinModeller.unitFile";
+	public final String STATSFILE = "JoinModeller.statsFile";
+	public final String MMFFILE = "JoinModeller.mmfFile";
+	public final String FULLFILE = "JoinModeller.fullFile";
+	public final String CXCHEDFILE = "JoinModeller.cxcJoinFile";
+	public final String JOINTREEFILE = "JoinModeller.joinTreeFile";
+	public final String CNVHEDFILE = "JoinModeller.cnvJoinFile";
+	public final String TRNCONFFILE = "JoinModeller.trnFile";
+	public final String CNVCONFFILE = "JoinModeller.cnvFile";
+	public final String HHEDCOMMAND = "JoinModeller.hhedCommand";
+	public final String FEATURELISTFILE = "JoinModeller.featureListFile";
+	public final String TRICKYPHONESFILE = "JoinModeller.trickyPhonesFile";
 
-            if (edge == vNoEdge && edgeNext == vNoEdge) {
-                // TODO: do we need this?
-                int phone = fv.getFeatureAsInt(iPhoneme);
-                assert 0 <= phone && phone < nPhonemes;
-                byte lr = fv.getByteFeature(iLeftRight);
-                int phoneNext = fvNext.getFeatureAsInt(iPhoneme);
-                byte lrNext = fvNext.getByteFeature(iLeftRight);
-                String pair = def.getFeatureValueAsString(iPhoneme, phone) +
-                    "_" + def.getFeatureValueAsString(iLeftRight, lr) +
-                    "-" +
-                    def.getFeatureValueAsString(iPhoneme, phoneNext) +
-                    "_" + def.getFeatureValueAsString(iLeftRight, lrNext);
+	public JoinModeller() {
 
-                // Compute the difference vector
-                float[] myRightFrame = joinFeatures.getRightJCF(i);
-                float[] nextLeftFrame = joinFeatures.getLeftJCF(i+1);
- 
-                double[] difference = new double[myRightFrame.length];                 
-                for (int k=0, len=myRightFrame.length; k<len; k++) {
-                    if(k==(len-1) && (myRightFrame[k] == Float.POSITIVE_INFINITY || nextLeftFrame[k] == Float.POSITIVE_INFINITY) ){
-                      // TODO: find out in JoinCostFeatureFileWriter why Infinity values occur, and fix it.
-                      difference[k] = 0.0;  
-                      System.out.println("WARNING: numUnit="+ i + " myRightFrame[k]="+myRightFrame[k] + " nextLeftFrame[k]=" + nextLeftFrame[k]);
-                    }
-                    else
-                      difference[k] = ((double)myRightFrame[k]) - nextLeftFrame[k];                   
-                 }   
-                
+	}
 
-                // Group the units with the same feature vectors
-                String contextName = contextTranslator.features2LongContext(def, fv, featureList);
-                Set<double[]> unitsWithFV = uniqueFeatureVectors.get(contextName);
-                if (unitsWithFV == null) {
-                    unitsWithFV = new HashSet<double[]>();
-                    uniqueFeatureVectors.put(contextName, unitsWithFV);
-                }
-                unitsWithFV.add(difference);
-            }
-        }
-        
-        int numUniqueFea = 1;
-        int n,i,j,m;
-        for (String fvString : uniqueFeatureVectors.keySet()) {
-            double[][] diffVectors = uniqueFeatureVectors.get(fvString).toArray(new double[0][]);
-            n = diffVectors.length;
-            
-            // Compute means and variances of the features across difference vectors         
-            double[] means;
-            double[] variances;
-            if (n == 1) {
-                means = diffVectors[0];
-                variances = MathUtils.zeros(means.length);
-               
-            } else {
-                means = MathUtils.mean(diffVectors, true);
-                variances = MathUtils.variance(diffVectors, means, true);
-      
-            }
-            
-            assert means.length == numFeatures : "expected to have " + numFeatures + " features, got " + means.length;            
-            
-            fullStream.write(fvString + "\n");
-            statsStream.write(numUniqueFea + " \"" + fvString + "\"    " + n + "    " + n + "\n");
-            
-            mmfStream.write("~h " + "\"" + fvString + "\"\n" );
+	public String getName() {
+		return "JoinModeller";
+	}
 
-            /* Stream 1*/
-            mmfStream.write("<BEGINHMM>\n<NUMSTATES> 3\n<STATE> 2\n");
-            mmfStream.write("<MEAN> " + numFeatures + "\n");
-            for (i=0; i<means.length; i++)
-              mmfStream.write(means[i] + " ");
-            mmfStream.write("\n<VARIANCE> " + numFeatures + "\n");
-            for (i=0; i<variances.length; i++)
-              mmfStream.write(variances[i] + " ");
-  
-            mmfStream.write("\n~t \"trP_1\"\n<ENDHMM>\n");
-            numUniqueFea++;
-        }
-        fullStream.close();
-        statsStream.close();
-        mmfStream.close();
-        
-        Process proc = null;
-        String cmdLine = null;
-        BufferedReader procStdout = null;
-        String line = null;
-        String filedir = db.getProp(db.FILEDIR);
-        
-        System.out.println(uniqueFeatureVectors.keySet().size() + " unique feature vectors, "+numUnits +" units");
-        System.out.println("Generated files: " + getProp(STATSFILE) + " " + getProp(MMFFILE) + " " + getProp(FULLFILE));
-        
-        System.out.println("\n---- Creating tree clustering command file for HHEd\n");
-        PrintWriter pw = new PrintWriter(new File(getProp(CXCHEDFILE)));
-        pw.println("// load stats file");
-        pw.println("RO 000 \"" + getProp(STATSFILE)+"\"");
-        pw.println();
-        pw.println("TR 0");
-        pw.println();
-        pw.println("// questions for decision tree-based context clustering");
-        for (String f : featureList) {               
-            int featureIndex = def.getFeatureIndex(f);
-            String[] values = def.getPossibleValues(featureIndex);
-            for (String v : values) {
-                if (f.endsWith("phone")) {
-                    v = contextTranslator.replaceTrickyPhones(v);
-                } else if (f.endsWith("sentence_punc") || f.endsWith("punctuation")) {
-                    v = contextTranslator.replacePunc(v);
-                }
-                pw.println("QS \""+f+"="+v+"\" {*|"+f+"="+v+"|*}");
-            }
-            pw.println();
-        }
-        pw.println("TR 3");
-        pw.println();
-        pw.println("// construct decision trees");
-        pw.println("TB 000 join_ {*.state[2]}");
-        pw.println();
-        pw.println("TR 1");
-        pw.println();
-        pw.println("// output constructed tree");
-        pw.println("ST \"" + getProp(JOINTREEFILE) + "\"");
-        pw.close();
-        
-        System.out.println("\n---- Tree-based context clustering for joinModeller\n");
-        // here the input file is join_mmf.mry and the output is  join_mmf.mry.clustered    
-        cmdLine = getProp(HHEDCOMMAND) + " -A -C " + getProp(TRNCONFFILE) + " -D -T 2 -p -i -H " + getProp(MMFFILE) + " -m -a 1.0 -w " + getProp(MMFFILE)+".clustered"  + " " + getProp(CXCHEDFILE) + " " + getProp(FULLFILE);
-        General.launchProc(cmdLine, "HHEd", filedir);
+	public SortedMap<String, String> getDefaultProps(DatabaseLayout db) {
+		this.db = db;
+		if (props == null) {
+			props = new TreeMap<String, String>();
+			String filedir = db.getProp(db.FILEDIR);
+			String maryExt = db.getProp(db.MARYEXT);
+			props.put(JOINCOSTFEATURESFILE, filedir + "joinCostFeatures" + maryExt);
+			props.put(UNITFEATURESFILE, filedir + "halfphoneFeatures" + maryExt);
+			props.put(UNITFILE, filedir + "halfphoneUnits" + maryExt);
+			props.put(STATSFILE, filedir + "stats" + maryExt);
+			props.put(MMFFILE, filedir + "join_mmf" + maryExt);
+			props.put(FULLFILE, filedir + "fullList" + maryExt);
+			props.put(FEATURELISTFILE, filedir + "/mary/featureListFile.txt");
+			props.put(TRICKYPHONESFILE, filedir + "/mary/trickyPhones.txt");
+			props.put(CXCHEDFILE, filedir + "cxc_join.hed");
+			props.put(JOINTREEFILE, filedir + "join_tree.inf");
+			props.put(CNVHEDFILE, filedir + "cnv_join.hed");
+			props.put(TRNCONFFILE, filedir + "trn.cnf");
+			props.put(CNVCONFFILE, filedir + "cnv.cnf");
+			props.put(HHEDCOMMAND, "/project/mary/marcela/sw/HTS_2.0.1/htk/bin/HHEd");
+		}
+		return props;
+	}
 
-        System.out.println("\n---- Creating conversion-to-hts command file for HHEd\n");
-        pw = new PrintWriter(new File(getProp(CNVHEDFILE)));
-        pw.println("TR 3");
-        pw.println();
-        pw.println("// load trees for joinModeller");
-        pw.println("LT \"" + getProp(JOINTREEFILE) + "\"");
-        pw.println();
-        pw.println("// convert loaded trees for hts_engine format");
-        pw.println("CT \""+filedir+"\"");
-        pw.println();
-        pw.println("// convert mmf for hts_engine format");
-        pw.println("CM \"" + filedir + "\"");
-        pw.close();
-        
-        
-        System.out.println("\n---- Converting mmfs to the hts_engine file format\n");
-        // the input of this command are: join_mmf.mry and join_tree.inf and the output: trees.1 and pdf.1
-        cmdLine = getProp(HHEDCOMMAND) + " -A -C " + getProp(CNVCONFFILE) + " -D -T 1 -p -i -H " + getProp(MMFFILE)+".clustered" + " " + getProp(CNVHEDFILE) + " " + getProp(FULLFILE);
-        General.launchProc(cmdLine, "HHEd", filedir);
-        
-        // the files trees.1 and pdf.1 produced by the previous command are renamed as tree-joinModeller.inf and joinModeller.pdf
-        cmdLine = "mv " + filedir + "trees.1 " + filedir + "tree-joinModeller.inf";
-        General.launchProc(cmdLine, "mv", filedir);
-        
-        cmdLine = "mv " + filedir + "pdf.1 " + filedir + "joinModeller.pdf";
-        General.launchProc(cmdLine, "mv", filedir);
-        
-        System.out.println("\n---- Created files: tree-joinModeller.inf, joinModeller.pdf");
-        
-        return true;
-    }
-    
-    /**
-     * Provide the progress of computation, in percent, or -1 if
-     * that feature is not implemented.
-     * @return -1 if not implemented, or an integer between 0 and 100.
-     */
-    public int getProgress()
-    {
-        return percent;
-    }
+	protected void setupHelp() {
+		props2Help = new TreeMap<String, String>();
+		props2Help.put(JOINCOSTFEATURESFILE, "file containing all halfphone units and their join cost features");
+		props2Help.put(UNITFEATURESFILE, "file containing all halfphone units and their target cost features");
+		props2Help.put(UNITFILE, "file containing all halfphone units");
+		props2Help.put(STATSFILE, "output file containing statistics of the models in HTK stats format");
+		props2Help
+				.put(MMFFILE,
+						"output file containing one state HMM models, HTK format, representing join models (mean and variances are calculated in this class)");
+		props2Help.put(FULLFILE, "output file containing the full list of HMM model names");
+		props2Help.put(FEATURELISTFILE, "feature list for making fullcontext names and questions");
+		props2Help.put(TRICKYPHONESFILE, "list of aliases for tricky phones, so HTK-HHEd command can handle them.");
+		props2Help
+				.put(CXCHEDFILE,
+						"HTK hed file used by HHEd, load stats file, contains questions for decision tree-based context clustering and outputs result in join-tree.inf");
+		props2Help.put(CNVHEDFILE, "HTK hed file used by HHEd to convert trees and mmf into hts_engine format");
+		props2Help.put(TRNCONFFILE, "HTK configuration file for context clustering");
+		props2Help.put(CNVCONFFILE, "HTK configuration file for converting to hts_engine format");
+		props2Help.put(HHEDCOMMAND, "HTS-HTK HHEd command, HTS version minimum HTS_2.0.1");
+	}
 
+	public boolean compute() throws IOException, Exception {
+		System.out.println("\n---- Training join models\n");
 
+		FeatureFileReader unitFeatures = FeatureFileReader.getFeatureFileReader(getProp(UNITFEATURESFILE));
+		JoinCostFeatures joinFeatures = new JoinCostFeatures(getProp(JOINCOSTFEATURESFILE));
+		UnitFileReader units = new UnitFileReader(getProp(UNITFILE));
+		FeatureDefinition def = unitFeatures.getFeatureDefinition();
 
-    
-    /** This function reads a feature list file originally used to train the HMMs, in HMM voices.
-     * The file format is simple, one feature after another like in the ../mary/features.txt
-     * An example of feature list is in ../mary/featuresHmmVoice.txt
-     * Since the features are provided by the user, it should be checked that the features exist
-     */
-    private void readFeatureList(String featureListFile, FeatureDefinition feaDef) throws Exception {
-      String line;
-      int i;
-  
-      //System.out.println("featureListFile: " + featureListFile);
-      Scanner s = null;
-      try {
-        s = new Scanner(new BufferedReader(new FileReader(featureListFile))).useDelimiter("\n");
-        String fea;
-        System.out.println("The following are other context features used for training Hmms: ");
-        while (s.hasNext()) {
-          fea = s.nextLine();
-          // Check if the feature exist
-          if( feaDef.hasFeature(fea)){
-            featureList.add(fea);
-            //System.out.println("  " + fea);
-          }
-          else
-            throw new Exception("Error: feature \"" + fea + "\" in feature list file: " + featureListFile + " does not exist in FeatureDefinition.");
-        }
-        if (s != null) { 
-          s.close();
-        }       
-      } catch (FileNotFoundException e) {
-          System.out.println("readFeatureList:  " + e.getMessage());
-      }      
-      System.out.println("readFeatureList: loaded " + featureList.size() + " context features from " + featureListFile);
-      
-    } /* method ReadFeatureList */
+		// for creating a contexttranslator object we need to check if there is
+		PhoneTranslator contextTranslator = null;
+		// Check if there are tricky phones
+		if (HMMVoiceMakeData.checkTrickyPhones(db.getAllophoneSet(), getProp(TRICKYPHONESFILE)))
+			contextTranslator = new PhoneTranslator(new FileInputStream(getProp(TRICKYPHONESFILE)));
+		else
+			contextTranslator = new PhoneTranslator(null);
+
+		featureList = new Vector<String>();
+		readFeatureList(getProp(FEATURELISTFILE), def);
+
+		statsStream = new FileWriter(getProp(STATSFILE));
+		mmfStream = new FileWriter(getProp(MMFFILE));
+		fullStream = new FileWriter(getProp(FULLFILE));
+		// output HTK model definition and dummy state-transition matrix, macro ~t "trP_1"
+		int numFeatures = joinFeatures.getNumberOfFeatures();
+		mmfStream.write("~o\n" + "<VECSIZE> " + numFeatures + " <USER><DIAGC>\n" + "~t \"trP_1\"\n<TRANSP> 3\n"
+				+ "0 1 0\n0 0 1\n0 0 0\n");
+
+		if (unitFeatures.getNumberOfUnits() != joinFeatures.getNumberOfUnits())
+			throw new IllegalStateException("Number of units in unit and join feature files does not match!");
+		if (unitFeatures.getNumberOfUnits() != units.getNumberOfUnits())
+			throw new IllegalStateException("Number of units in unit file and unit feature file does not match!");
+		int numUnits = unitFeatures.getNumberOfUnits();
+		// ----FeatureDefinition def = unitFeatures.getFeatureDefinition();
+
+		int iPhoneme = def.getFeatureIndex("phone");
+		int nPhonemes = def.getNumberOfValues(iPhoneme);
+		int iLeftRight = def.getFeatureIndex("halfphone_lr");
+		byte vLeft = def.getFeatureValueAsByte(iLeftRight, "L");
+		byte vRight = def.getFeatureValueAsByte(iLeftRight, "R");
+		int iEdge = def.getFeatureIndex("edge");
+		byte vNoEdge = def.getFeatureValueAsByte(iEdge, "0");
+		byte vStartEdge = def.getFeatureValueAsByte(iEdge, "start");
+		byte vEndEdge = def.getFeatureValueAsByte(iEdge, "end");
+
+		Map<String, Set<double[]>> uniqueFeatureVectors = new HashMap<String, Set<double[]>>();
+
+		// Now look at all pairs of adjacent units
+		FeatureVector fvNext = unitFeatures.getFeatureVector(0);
+		for (int i = 0; i < numUnits - 1; i++) {
+			percent = 100 * (i + 1) / numUnits;
+
+			FeatureVector fv = fvNext;
+			byte edge = fv.getByteFeature(iEdge);
+			fvNext = unitFeatures.getFeatureVector(i + 1);
+			byte edgeNext = fvNext.getByteFeature(iEdge);
+
+			if (edge == vNoEdge && edgeNext == vNoEdge) {
+				// TODO: do we need this?
+				int phone = fv.getFeatureAsInt(iPhoneme);
+				assert 0 <= phone && phone < nPhonemes;
+				byte lr = fv.getByteFeature(iLeftRight);
+				int phoneNext = fvNext.getFeatureAsInt(iPhoneme);
+				byte lrNext = fvNext.getByteFeature(iLeftRight);
+				String pair = def.getFeatureValueAsString(iPhoneme, phone) + "_" + def.getFeatureValueAsString(iLeftRight, lr)
+						+ "-" + def.getFeatureValueAsString(iPhoneme, phoneNext) + "_"
+						+ def.getFeatureValueAsString(iLeftRight, lrNext);
+
+				// Compute the difference vector
+				float[] myRightFrame = joinFeatures.getRightJCF(i);
+				float[] nextLeftFrame = joinFeatures.getLeftJCF(i + 1);
+
+				double[] difference = new double[myRightFrame.length];
+				for (int k = 0, len = myRightFrame.length; k < len; k++) {
+					if (k == (len - 1)
+							&& (myRightFrame[k] == Float.POSITIVE_INFINITY || nextLeftFrame[k] == Float.POSITIVE_INFINITY)) {
+						// TODO: find out in JoinCostFeatureFileWriter why Infinity values occur, and fix it.
+						difference[k] = 0.0;
+						System.out.println("WARNING: numUnit=" + i + " myRightFrame[k]=" + myRightFrame[k] + " nextLeftFrame[k]="
+								+ nextLeftFrame[k]);
+					} else
+						difference[k] = ((double) myRightFrame[k]) - nextLeftFrame[k];
+				}
+
+				// Group the units with the same feature vectors
+				String contextName = contextTranslator.features2LongContext(def, fv, featureList);
+				Set<double[]> unitsWithFV = uniqueFeatureVectors.get(contextName);
+				if (unitsWithFV == null) {
+					unitsWithFV = new HashSet<double[]>();
+					uniqueFeatureVectors.put(contextName, unitsWithFV);
+				}
+				unitsWithFV.add(difference);
+			}
+		}
+
+		int numUniqueFea = 1;
+		int n, i, j, m;
+		for (String fvString : uniqueFeatureVectors.keySet()) {
+			double[][] diffVectors = uniqueFeatureVectors.get(fvString).toArray(new double[0][]);
+			n = diffVectors.length;
+
+			// Compute means and variances of the features across difference vectors
+			double[] means;
+			double[] variances;
+			if (n == 1) {
+				means = diffVectors[0];
+				variances = MathUtils.zeros(means.length);
+
+			} else {
+				means = MathUtils.mean(diffVectors, true);
+				variances = MathUtils.variance(diffVectors, means, true);
+
+			}
+
+			assert means.length == numFeatures : "expected to have " + numFeatures + " features, got " + means.length;
+
+			fullStream.write(fvString + "\n");
+			statsStream.write(numUniqueFea + " \"" + fvString + "\"    " + n + "    " + n + "\n");
+
+			mmfStream.write("~h " + "\"" + fvString + "\"\n");
+
+			/* Stream 1 */
+			mmfStream.write("<BEGINHMM>\n<NUMSTATES> 3\n<STATE> 2\n");
+			mmfStream.write("<MEAN> " + numFeatures + "\n");
+			for (i = 0; i < means.length; i++)
+				mmfStream.write(means[i] + " ");
+			mmfStream.write("\n<VARIANCE> " + numFeatures + "\n");
+			for (i = 0; i < variances.length; i++)
+				mmfStream.write(variances[i] + " ");
+
+			mmfStream.write("\n~t \"trP_1\"\n<ENDHMM>\n");
+			numUniqueFea++;
+		}
+		fullStream.close();
+		statsStream.close();
+		mmfStream.close();
+
+		Process proc = null;
+		String cmdLine = null;
+		BufferedReader procStdout = null;
+		String line = null;
+		String filedir = db.getProp(db.FILEDIR);
+
+		System.out.println(uniqueFeatureVectors.keySet().size() + " unique feature vectors, " + numUnits + " units");
+		System.out.println("Generated files: " + getProp(STATSFILE) + " " + getProp(MMFFILE) + " " + getProp(FULLFILE));
+
+		System.out.println("\n---- Creating tree clustering command file for HHEd\n");
+		PrintWriter pw = new PrintWriter(new File(getProp(CXCHEDFILE)));
+		pw.println("// load stats file");
+		pw.println("RO 000 \"" + getProp(STATSFILE) + "\"");
+		pw.println();
+		pw.println("TR 0");
+		pw.println();
+		pw.println("// questions for decision tree-based context clustering");
+		for (String f : featureList) {
+			int featureIndex = def.getFeatureIndex(f);
+			String[] values = def.getPossibleValues(featureIndex);
+			for (String v : values) {
+				if (f.endsWith("phone")) {
+					v = contextTranslator.replaceTrickyPhones(v);
+				} else if (f.endsWith("sentence_punc") || f.endsWith("punctuation")) {
+					v = contextTranslator.replacePunc(v);
+				}
+				pw.println("QS \"" + f + "=" + v + "\" {*|" + f + "=" + v + "|*}");
+			}
+			pw.println();
+		}
+		pw.println("TR 3");
+		pw.println();
+		pw.println("// construct decision trees");
+		pw.println("TB 000 join_ {*.state[2]}");
+		pw.println();
+		pw.println("TR 1");
+		pw.println();
+		pw.println("// output constructed tree");
+		pw.println("ST \"" + getProp(JOINTREEFILE) + "\"");
+		pw.close();
+
+		System.out.println("\n---- Tree-based context clustering for joinModeller\n");
+		// here the input file is join_mmf.mry and the output is join_mmf.mry.clustered
+		cmdLine = getProp(HHEDCOMMAND) + " -A -C " + getProp(TRNCONFFILE) + " -D -T 2 -p -i -H " + getProp(MMFFILE)
+				+ " -m -a 1.0 -w " + getProp(MMFFILE) + ".clustered" + " " + getProp(CXCHEDFILE) + " " + getProp(FULLFILE);
+		General.launchProc(cmdLine, "HHEd", filedir);
+
+		System.out.println("\n---- Creating conversion-to-hts command file for HHEd\n");
+		pw = new PrintWriter(new File(getProp(CNVHEDFILE)));
+		pw.println("TR 3");
+		pw.println();
+		pw.println("// load trees for joinModeller");
+		pw.println("LT \"" + getProp(JOINTREEFILE) + "\"");
+		pw.println();
+		pw.println("// convert loaded trees for hts_engine format");
+		pw.println("CT \"" + filedir + "\"");
+		pw.println();
+		pw.println("// convert mmf for hts_engine format");
+		pw.println("CM \"" + filedir + "\"");
+		pw.close();
+
+		System.out.println("\n---- Converting mmfs to the hts_engine file format\n");
+		// the input of this command are: join_mmf.mry and join_tree.inf and the output: trees.1 and pdf.1
+		cmdLine = getProp(HHEDCOMMAND) + " -A -C " + getProp(CNVCONFFILE) + " -D -T 1 -p -i -H " + getProp(MMFFILE)
+				+ ".clustered" + " " + getProp(CNVHEDFILE) + " " + getProp(FULLFILE);
+		General.launchProc(cmdLine, "HHEd", filedir);
+
+		// the files trees.1 and pdf.1 produced by the previous command are renamed as tree-joinModeller.inf and joinModeller.pdf
+		cmdLine = "mv " + filedir + "trees.1 " + filedir + "tree-joinModeller.inf";
+		General.launchProc(cmdLine, "mv", filedir);
+
+		cmdLine = "mv " + filedir + "pdf.1 " + filedir + "joinModeller.pdf";
+		General.launchProc(cmdLine, "mv", filedir);
+
+		System.out.println("\n---- Created files: tree-joinModeller.inf, joinModeller.pdf");
+
+		return true;
+	}
+
+	/**
+	 * Provide the progress of computation, in percent, or -1 if that feature is not implemented.
+	 * 
+	 * @return -1 if not implemented, or an integer between 0 and 100.
+	 */
+	public int getProgress() {
+		return percent;
+	}
+
+	/**
+	 * This function reads a feature list file originally used to train the HMMs, in HMM voices. The file format is simple, one
+	 * feature after another like in the ../mary/features.txt An example of feature list is in ../mary/featuresHmmVoice.txt Since
+	 * the features are provided by the user, it should be checked that the features exist
+	 */
+	private void readFeatureList(String featureListFile, FeatureDefinition feaDef) throws Exception {
+		String line;
+		int i;
+
+		// System.out.println("featureListFile: " + featureListFile);
+		Scanner s = null;
+		try {
+			s = new Scanner(new BufferedReader(new FileReader(featureListFile))).useDelimiter("\n");
+			String fea;
+			System.out.println("The following are other context features used for training Hmms: ");
+			while (s.hasNext()) {
+				fea = s.nextLine();
+				// Check if the feature exist
+				if (feaDef.hasFeature(fea)) {
+					featureList.add(fea);
+					// System.out.println("  " + fea);
+				} else
+					throw new Exception("Error: feature \"" + fea + "\" in feature list file: " + featureListFile
+							+ " does not exist in FeatureDefinition.");
+			}
+			if (s != null) {
+				s.close();
+			}
+		} catch (FileNotFoundException e) {
+			System.out.println("readFeatureList:  " + e.getMessage());
+		}
+		System.out.println("readFeatureList: loaded " + featureList.size() + " context features from " + featureListFile);
+
+	} /* method ReadFeatureList */
+
+	public static void main(String[] args) throws IOException, InterruptedException {
+		try {
+			/* configure log info */
+			org.apache.log4j.BasicConfigurator.configure();
+
+			/* These trees are for halphone features */
+			String joinPdfFile = "/project/mary/marcela/HMM-voices/DFKI_German_Poker/mary_files_old/joinModeller.pdf";
+			String joinTreeFile = "/project/mary/marcela/HMM-voices/DFKI_German_Poker/mary_files_old/tree-joinModeller.inf";
+
+			// String contextFile = "/project/mary/marcela/HMM-voices/DFKI_German_Poker/phonefeatures/m0001.pfeats";
+			/* halphone example */
+			String contextFile = "/project/mary/marcela/unitselection-halfphone.pfeats";
+			Scanner context = new Scanner(new BufferedReader(new FileReader(contextFile)));
+			String strContext = "";
+			while (context.hasNext()) {
+				strContext += context.nextLine();
+				strContext += "\n";
+			}
+			context.close();
+			// System.out.println(strContext);
+			FeatureDefinition def = new FeatureDefinition(new BufferedReader(new StringReader(strContext)), false);
+
+			String trickyPhonesFile = "/project/mary/marcela/HMM-voices/DFKI_German_Poker/mary_files_old/trickyPhones.txt";
+			String allophonesFile = "/project/mary/marcela/openmary/lib/modules/en/us/lexicon/allophones.en_US.xml";
+			// Check if there are tricky phones
+			if (!(HMMVoiceMakeData.checkTrickyPhones(AllophoneSet.getAllophoneSet(allophonesFile), trickyPhonesFile)))
+				trickyPhonesFile = null;
+
+			// Check if there are tricky phones, and create a PhoneTranslator object
+			PhoneTranslator phTranslator = new PhoneTranslator(new FileInputStream(trickyPhonesFile));
+
+			CART[] joinTree = null;
+			HTSCARTReader htsReader = new HTSCARTReader();
+			int numStates = 1; // just one state in the joinModeller
+			int vectorSize = 0;
+
+			try {
+				joinTree = htsReader.load(numStates, new FileInputStream(joinTreeFile), new FileInputStream(joinPdfFile),
+						PdfFileFormat.join, def, phTranslator);
+				vectorSize = htsReader.getVectorSize();
+
+			} catch (Exception e) {
+				IOException ioe = new IOException("Cannot load join model trees from " + joinTreeFile);
+				ioe.initCause(e);
+				throw ioe;
+			}
+
+			/* Given a context feature model name, find its join PDF mean and variance */
+			/* first, find an index in the tree and then find the mean and variance that corresponds to that index in joinPdf */
+			double[] mean = new double[vectorSize];
+			double[] variance = new double[vectorSize];
+
+			// String modelName =
+			// "0^f-t+_=0||str=0|pos_syl=3|pos_type=final|pos=NN|snt_punc=.|snt_numwrds=7|wrds_snt_stt=2|wrds_snt_end=4|wrd_numsyls=3|syls_wrd_stt=2|syls_wrd_end=0|wrd_numsegs=9|segs_wrd_stt=8|segs_wrd_end=0|syl_numsegs=4|segs_syl_stt=3|segs_syl_end=0|syls_p_str=2|syls_n_str=1|p_punc=0|n_punc=.|wrds_p_punc=2|wrds_n_punc=4|wrd_freq=0|";
+			// String modelName =
+			// "0^i:-o:+_=0||str=0|pos_syl=0|pos_type=final|pos=FM|snt_punc=.|snt_numwrds=17|wrds_snt_stt=16|wrds_snt_end=0|wrd_numsyls=4|syls_wrd_stt=3|syls_wrd_end=0|wrd_numsegs=7|segs_wrd_stt=6|segs_wrd_end=0|syl_numsegs=1|segs_syl_stt=0|segs_syl_end=0|syls_p_str=2|syls_n_str=0|p_punc=:|n_punc=.|wrds_p_punc=4|wrds_n_punc=0|wrd_freq=0|";
+			// String modelName =
+			// "aI^R-a:+t=@||wrds_n_punc=0|syl_numsegs=2|segs_wrd_stt=5|syls_wrd_stt=2|str=0|wrds_snt_end=0|pos_syl=1|p_punc=0|wrd_freq=3|snt_punc=dt|segs_wrd_end=3|wrds_snt_stt=4|wrds_p_punc=4|n_punc=dt|segs_syl_end=0|syls_wrd_end=1|pos_type=mid|syls_n_str=0|POS=VVPP|snt_numwrds=5|wrd_numsyls=4|wrd_numsegs=9|syls_p_str=1|segs_syl_stt=1||";
+			// String feaLine =
+			// "24 0 1 0 0 1 2 3 1 0 30 2 2 1 1 0 0 0 0 43 2 2 4 1 0 0 0 0 1 1 0 1 3 0 0 2 2 0 0 0 0 0 1 1 1 0 0 5 1 3 5 1 3 10 1 9 1 3 1 1 3 0 2 6 1 4 2 1 0 1 3 1 7 9 0 1 6 0 1 1 3 0";
+			// String feaLine =
+			// "23 0 2 0 0 2 0 0 0 0 0 0 0 0 0 0 0 0 0 34 2 4 2 2 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0";
+			/* halfphone example */
+			String feaLine = "8 3 2 0 1 0 6 1 0 0 4 2 38 1 0 0 1 1 1 2 2 35 2 3 1 1 0 0 0 0 1 2 1 0 1 0 0 13 0 0 0 0 0 0 0 0 1 0 0 6 0 5 6 0 5 9 0 9 0 5 0 1 1 0 0 3 2 0 3 2 0 0 1 0 7 9 0 2 0 0 0 0 6 8";
+			FeatureVector fv;
+
+			context = new Scanner(new BufferedReader(new FileReader(contextFile)));
+			while (context.hasNext()) {
+				feaLine = context.nextLine();
+				if (feaLine.trim().equals(""))
+					break;
+			}
+			while (context.hasNext()) {
+				feaLine = context.nextLine();
+				if (feaLine.trim().equals(""))
+					break;
+			}
+			while (context.hasNext()) {
+				feaLine = context.nextLine();
+				// System.out.println(feaLine);
+
+				fv = def.toFeatureVector(0, feaLine);
+
+				Node node = joinTree[0].interpretToNode(fv, 1);
+				assert node instanceof PdfLeafNode : "The node must be a PdfLeafNode.";
+
+				mean = ((PdfLeafNode) node).getMean();
+				variance = ((PdfLeafNode) node).getVariance();
+
+				System.out.print("mean: ");
+				for (int i = 0; i < vectorSize; i++)
+					System.out.print(mean[i] + " ");
+				System.out.print("\nvariance: ");
+				for (int i = 0; i < vectorSize; i++)
+					System.out.print(variance[i] + " ");
+				System.out.println();
+			}
+
+		} catch (Exception e) {
+			System.err.println("Exception: " + e.getMessage());
+		}
+
+	}
 
- 
-    
-    public static void main(String[] args) throws IOException, InterruptedException{
-        try {
-            /* configure log info */
-            org.apache.log4j.BasicConfigurator.configure();
-            
-            /* These trees are for halphone features */
-            String joinPdfFile = "/project/mary/marcela/HMM-voices/DFKI_German_Poker/mary_files_old/joinModeller.pdf";
-            String joinTreeFile = "/project/mary/marcela/HMM-voices/DFKI_German_Poker/mary_files_old/tree-joinModeller.inf";
-  
-            //String contextFile = "/project/mary/marcela/HMM-voices/DFKI_German_Poker/phonefeatures/m0001.pfeats";
-            /*halphone example */
-            String contextFile = "/project/mary/marcela/unitselection-halfphone.pfeats";
-            Scanner context = new Scanner(new BufferedReader(new FileReader(contextFile)));
-            String strContext="";
-            while (context.hasNext()) {
-              strContext += context.nextLine(); 
-              strContext += "\n";
-            }
-            context.close();
-            //System.out.println(strContext);
-            FeatureDefinition def = new FeatureDefinition(new BufferedReader(new StringReader(strContext)), false);
-            
-            String trickyPhonesFile = "/project/mary/marcela/HMM-voices/DFKI_German_Poker/mary_files_old/trickyPhones.txt";
-            String allophonesFile = "/project/mary/marcela/openmary/lib/modules/en/us/lexicon/allophones.en_US.xml";
-            // Check if there are tricky phones          
-            if( ! (HMMVoiceMakeData.checkTrickyPhones(AllophoneSet.getAllophoneSet(allophonesFile), trickyPhonesFile) ) )
-                trickyPhonesFile = null;
-             
-            // Check if there are tricky phones, and create a PhoneTranslator object
-            PhoneTranslator phTranslator = new PhoneTranslator(new FileInputStream(trickyPhonesFile));
-  
-            CART[] joinTree = null;
-            HTSCARTReader htsReader = new HTSCARTReader();
-            int numStates = 1;  // just one state in the joinModeller
-            int vectorSize = 0;
-            
-            try {
-                joinTree = htsReader.load(numStates, new FileInputStream(joinTreeFile), new FileInputStream(joinPdfFile), PdfFileFormat.join, def, phTranslator);
-                vectorSize = htsReader.getVectorSize();
-                
-            } catch (Exception e) {
-                IOException ioe = new IOException("Cannot load join model trees from "+joinTreeFile);
-                ioe.initCause(e);
-                throw ioe;
-            }
-            
-            
-            /* Given a context feature model name, find its join PDF mean and variance */
-            /* first, find an index in the tree and then find the mean and variance that corresponds to that index in joinPdf */
-            double[] mean = new double[vectorSize];
-            double[] variance = new double[vectorSize];
-            
-            //String modelName = "0^f-t+_=0||str=0|pos_syl=3|pos_type=final|pos=NN|snt_punc=.|snt_numwrds=7|wrds_snt_stt=2|wrds_snt_end=4|wrd_numsyls=3|syls_wrd_stt=2|syls_wrd_end=0|wrd_numsegs=9|segs_wrd_stt=8|segs_wrd_end=0|syl_numsegs=4|segs_syl_stt=3|segs_syl_end=0|syls_p_str=2|syls_n_str=1|p_punc=0|n_punc=.|wrds_p_punc=2|wrds_n_punc=4|wrd_freq=0|";
-            //String modelName = "0^i:-o:+_=0||str=0|pos_syl=0|pos_type=final|pos=FM|snt_punc=.|snt_numwrds=17|wrds_snt_stt=16|wrds_snt_end=0|wrd_numsyls=4|syls_wrd_stt=3|syls_wrd_end=0|wrd_numsegs=7|segs_wrd_stt=6|segs_wrd_end=0|syl_numsegs=1|segs_syl_stt=0|segs_syl_end=0|syls_p_str=2|syls_n_str=0|p_punc=:|n_punc=.|wrds_p_punc=4|wrds_n_punc=0|wrd_freq=0|";
-            //String modelName = "aI^R-a:+t=@||wrds_n_punc=0|syl_numsegs=2|segs_wrd_stt=5|syls_wrd_stt=2|str=0|wrds_snt_end=0|pos_syl=1|p_punc=0|wrd_freq=3|snt_punc=dt|segs_wrd_end=3|wrds_snt_stt=4|wrds_p_punc=4|n_punc=dt|segs_syl_end=0|syls_wrd_end=1|pos_type=mid|syls_n_str=0|POS=VVPP|snt_numwrds=5|wrd_numsyls=4|wrd_numsegs=9|syls_p_str=1|segs_syl_stt=1||";
-            //String feaLine = "24 0 1 0 0 1 2 3 1 0 30 2 2 1 1 0 0 0 0 43 2 2 4 1 0 0 0 0 1 1 0 1 3 0 0 2 2 0 0 0 0 0 1 1 1 0 0 5 1 3 5 1 3 10 1 9 1 3 1 1 3 0 2 6 1 4 2 1 0 1 3 1 7 9 0 1 6 0 1 1 3 0";
-            //String feaLine = "23 0 2 0 0 2 0 0 0 0 0 0 0 0 0 0 0 0 0 34 2 4 2 2 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0";
-            /* halfphone example */
-            String feaLine = "8 3 2 0 1 0 6 1 0 0 4 2 38 1 0 0 1 1 1 2 2 35 2 3 1 1 0 0 0 0 1 2 1 0 1 0 0 13 0 0 0 0 0 0 0 0 1 0 0 6 0 5 6 0 5 9 0 9 0 5 0 1 1 0 0 3 2 0 3 2 0 0 1 0 7 9 0 2 0 0 0 0 6 8";
-            FeatureVector fv;
-            
-            context = new Scanner(new BufferedReader(new FileReader(contextFile)));
-            while (context.hasNext()) {
-              feaLine = context.nextLine(); 
-              if (feaLine.trim().equals("")) break;
-            }
-            while (context.hasNext()) {
-                feaLine = context.nextLine(); 
-                if (feaLine.trim().equals("")) break;
-            }
-            while (context.hasNext()) {
-                feaLine = context.nextLine(); 
-                //System.out.println(feaLine);
-            
-            fv = def.toFeatureVector(0, feaLine);
-            
-            Node node = joinTree[0].interpretToNode(fv, 1);
-            assert node instanceof PdfLeafNode : "The node must be a PdfLeafNode.";
-            
-            mean = ((PdfLeafNode)node).getMean();
-            variance = ((PdfLeafNode)node).getVariance();
-            
-            System.out.print("mean: ");
-            for(int i=0; i<vectorSize; i++)
-               System.out.print(mean[i] + " ");
-            System.out.print("\nvariance: ");
-            for(int i=0; i<vectorSize; i++)
-               System.out.print(variance[i] + " ");
-            System.out.println();
-            }
-            
-        } catch (Exception e) {
-            System.err.println("Exception: " + e.getMessage());
-        }
-        
-      }
-    
-    
 }

marytts-builder/src/main/java/marytts/tools/voiceimport/LabelPauseDeleter.java

@@ -27,220 +27,209 @@ import java.util.SortedMap;
 import java.util.StringTokenizer;
 import java.util.TreeMap;
 
-
 public class LabelPauseDeleter extends VoiceImportComponent {
-        
-        private DatabaseLayout db;        
-        private File ehmm;
-        private String outputDir;
-        protected String labExt = ".lab";
-        protected String pauseSymbol;
-        private int progress;
-        private String locale;
-        
-        public final String EDIR = "LabelPauseDeleter.eDir";
-        public final String OUTLABDIR = "LabelPauseDeleter.outputLabDir";
-        public final String PAUSETHR = "LabelPauseDeleter.pauseDurationThreshold";
-        
-        public final String getName(){
-            return "LabelPauseDeleter";
-        }
-        
-       public SortedMap getDefaultProps(DatabaseLayout db){
-           this.db = db;
-           if (props == null){
-               props = new TreeMap();
-               String ehmmdir = System.getProperty("EHMMDIR");
-               if ( ehmmdir == null ) {
-                   ehmmdir = "/project/mary/Festival/festvox/src/ehmm/";
-               }
-               
-               props.put(EDIR,db.getProp(db.ROOTDIR)
-                            +"ehmm"
-                            +System.getProperty("file.separator"));
-               props.put(OUTLABDIR, db.getProp(db.ROOTDIR)
-                       +"lab"
-                       +System.getProperty("file.separator"));
-               props.put(PAUSETHR, "100");
-           }
-           return props;
-       }
-       
-       protected void setupHelp(){
-           props2Help = new TreeMap();
-           props2Help.put(EDIR,"directory containing all files used for training and labeling.");
-           props2Help.put(OUTLABDIR, "Directory to store generated lebels from EHMM.");
-           props2Help.put(PAUSETHR, "Threshold for deleting pauses from label files");
-        }
-        
-       @Override
-       protected void initialiseComp()
-        {
-           locale = db.getProp(db.LOCALE);
-           this.pauseSymbol = System.getProperty("pause.symbol", "_");
-        }
-        
-        /**
-         * Do the computations required by this component.
-         * 
-         * @return true on success, false on failure
-         */
-        public boolean compute() throws Exception{
-            
-                                    
-            ehmm = new File(getProp(EDIR));
-            System.out.println("Copying label files into lab directory ...");
-            getProperLabelFormat();
-            System.out.println(" ... done.");
-            
-            return true;
-        }
-        
-         
-        /**
-         * Post processing Step to convert Label files
-         * to MARY supportable format
-         * @throws Exception
-         */        
-        private void getProperLabelFormat() throws Exception {
-            
-            List<String> problems = new ArrayList<String>();
-            for (int i=0; i<bnl.getLength(); i++) {
-                progress = 100*i/bnl.getLength();
-                String basename = bnl.getName(i);
-                boolean ok = convertSingleLabelFile(basename);
-                if (ok) {
-                    System.out.println( "    " + basename);
-                } else {
-                    System.out.println("     cannot read " + basename);
-                    problems.add(basename);
-                }
-            }
-            if (problems.size() > 0) {
-                System.out.println(problems.size() + " out of " + bnl.getLength() + " could not be read:");
-                for (String b : problems) {
-                    System.out.println("    "+b);
-                }
-            }
-        }
-        
-       
-        
-        /**
-         * Post Processing single Label file
-         * @param basename
-         * @throws Exception
-         * @return true on success, false on failure
-         */
-        private boolean convertSingleLabelFile(String basename) throws Exception {
-            
-            String line;
-            String previous, current;
-            String regexp1 = "pau";
-            String regexp2 = "ssil";
-            
-            File labDir = new File(getProp(OUTLABDIR));
-            if(!labDir.exists()){
-                labDir.mkdir();
-            }
-            
-            // READ LABEL FILE
-            String filename = getProp(EDIR)+"/lab/"+basename+labExt;
-            if (!new File(filename).exists()) return false;
-            try {
-                UnitLabel[] ulab = UnitLabel.readLabFile(filename);
-                
-                // Remove multiple consecutive pauses
-                ArrayList<UnitLabel> arrayLabel = new ArrayList<UnitLabel>();
-                for(int i=0; i<ulab.length; i++){
-                    boolean iscPause = ulab[i].getUnitName().matches(regexp1) || ulab[i].getUnitName().matches(regexp2);
-                    if((i+1)<ulab.length){
-                        boolean isnPause = ulab[i+1].getUnitName().matches(regexp1) || ulab[i+1].getUnitName().matches(regexp2);
-                        if(iscPause && isnPause){
-                            ulab[i+1].setStartTime(ulab[i].getStartTime());
-                            //System.out.println(ulab[i].getEndTime()+" "+ulab[i].getUnitIndex()+" "+ulab[i].getUnitName());
-                            continue;
-                        }
-                    }
-                    if(iscPause){
-                        ulab[i].setUnitName(pauseSymbol);
-                    }
-                    arrayLabel.add(ulab[i]);
-                }
-                
-                // Remove pauses below given threshold 
-                for(int i=0; i<arrayLabel.size(); i++){
-                    UnitLabel ul = arrayLabel.get(i);
-                    if(i>0 && (i+1)<arrayLabel.size()){
-                        if(ul.getUnitName().equals(pauseSymbol)){
-                            double duration = ul.endTime - ul.startTime;
-                            if(!isRealPause(duration)){
-                                System.out.println("deleting... " + ul.startTime + " "+ ul.endTime + " "+ul.unitName);
-                                UnitLabel pul = arrayLabel.get(i-1);
-                                UnitLabel nul = arrayLabel.get(i+1);
-                                pul.setEndTime(pul.getEndTime()+((double)duration / 2.0));
-                                nul.setStartTime(nul.getStartTime()-((double)duration / 2.0));
-                                arrayLabel.remove(i--);
-                            }
-                        }
-                    }
-                }
-                ulab = arrayLabel.toArray(new UnitLabel[0]);
-                
-                // write labels into given file
-                UnitLabel.writeLabFile(ulab, labDir+"/"+basename+labExt);
-            } catch (Exception e) {
-                e.printStackTrace();
-                return false;
-            }
-            return true; // success
 
-        }
-      
-      private boolean isRealPause(double phoneDuration){
-          /*TODO:
-           * Here we need to modify thresholds
-           */
-          
-          double threshold = Double.parseDouble(getProp(PAUSETHR)) ;
-          if(phoneDuration > (threshold / (double) 1000.0)){
-              return true;
-          }
-          
-          return false;
-      }
-      
-         
-     /**
-      * To get Label Unit DATA (time stamp, index, phone unit)
-      * @param line
-      * @return ArrayList contains time stamp, index and phone unit
-      * @throws IOException
-      */
-        private ArrayList getLabelUnitData(String line) throws IOException
-        {
-            if (line == null) return null;
-            ArrayList unitData = new ArrayList();
-            StringTokenizer st = new StringTokenizer(line.trim());
-           //the first token is the time
-            unitData.add(st.nextToken()); 
-           //the second token is the unit index
-            unitData.add(st.nextToken());
-           //the third token is the phone
-           unitData.add(st.nextToken());
-           return unitData;
-        }
-        
-            
-        /**
-         * Provide the progress of computation, in percent, or -1 if
-         * that feature is not implemented.
-         * @return -1 if not implemented, or an integer between 0 and 100.
-         */
-        public int getProgress()
-        {
-            return progress;
-        }
+	private DatabaseLayout db;
+	private File ehmm;
+	private String outputDir;
+	protected String labExt = ".lab";
+	protected String pauseSymbol;
+	private int progress;
+	private String locale;
+
+	public final String EDIR = "LabelPauseDeleter.eDir";
+	public final String OUTLABDIR = "LabelPauseDeleter.outputLabDir";
+	public final String PAUSETHR = "LabelPauseDeleter.pauseDurationThreshold";
+
+	public final String getName() {
+		return "LabelPauseDeleter";
+	}
+
+	public SortedMap getDefaultProps(DatabaseLayout db) {
+		this.db = db;
+		if (props == null) {
+			props = new TreeMap();
+			String ehmmdir = System.getProperty("EHMMDIR");
+			if (ehmmdir == null) {
+				ehmmdir = "/project/mary/Festival/festvox/src/ehmm/";
+			}
+
+			props.put(EDIR, db.getProp(db.ROOTDIR) + "ehmm" + System.getProperty("file.separator"));
+			props.put(OUTLABDIR, db.getProp(db.ROOTDIR) + "lab" + System.getProperty("file.separator"));
+			props.put(PAUSETHR, "100");
+		}
+		return props;
+	}
+
+	protected void setupHelp() {
+		props2Help = new TreeMap();
+		props2Help.put(EDIR, "directory containing all files used for training and labeling.");
+		props2Help.put(OUTLABDIR, "Directory to store generated lebels from EHMM.");
+		props2Help.put(PAUSETHR, "Threshold for deleting pauses from label files");
+	}
+
+	@Override
+	protected void initialiseComp() {
+		locale = db.getProp(db.LOCALE);
+		this.pauseSymbol = System.getProperty("pause.symbol", "_");
+	}
+
+	/**
+	 * Do the computations required by this component.
+	 * 
+	 * @return true on success, false on failure
+	 */
+	public boolean compute() throws Exception {
+
+		ehmm = new File(getProp(EDIR));
+		System.out.println("Copying label files into lab directory ...");
+		getProperLabelFormat();
+		System.out.println(" ... done.");
+
+		return true;
+	}
+
+	/**
+	 * Post processing Step to convert Label files to MARY supportable format
+	 * 
+	 * @throws Exception
+	 */
+	private void getProperLabelFormat() throws Exception {
+
+		List<String> problems = new ArrayList<String>();
+		for (int i = 0; i < bnl.getLength(); i++) {
+			progress = 100 * i / bnl.getLength();
+			String basename = bnl.getName(i);
+			boolean ok = convertSingleLabelFile(basename);
+			if (ok) {
+				System.out.println("    " + basename);
+			} else {
+				System.out.println("     cannot read " + basename);
+				problems.add(basename);
+			}
+		}
+		if (problems.size() > 0) {
+			System.out.println(problems.size() + " out of " + bnl.getLength() + " could not be read:");
+			for (String b : problems) {
+				System.out.println("    " + b);
+			}
+		}
+	}
+
+	/**
+	 * Post Processing single Label file
+	 * 
+	 * @param basename
+	 * @throws Exception
+	 * @return true on success, false on failure
+	 */
+	private boolean convertSingleLabelFile(String basename) throws Exception {
+
+		String line;
+		String previous, current;
+		String regexp1 = "pau";
+		String regexp2 = "ssil";
+
+		File labDir = new File(getProp(OUTLABDIR));
+		if (!labDir.exists()) {
+			labDir.mkdir();
+		}
+
+		// READ LABEL FILE
+		String filename = getProp(EDIR) + "/lab/" + basename + labExt;
+		if (!new File(filename).exists())
+			return false;
+		try {
+			UnitLabel[] ulab = UnitLabel.readLabFile(filename);
+
+			// Remove multiple consecutive pauses
+			ArrayList<UnitLabel> arrayLabel = new ArrayList<UnitLabel>();
+			for (int i = 0; i < ulab.length; i++) {
+				boolean iscPause = ulab[i].getUnitName().matches(regexp1) || ulab[i].getUnitName().matches(regexp2);
+				if ((i + 1) < ulab.length) {
+					boolean isnPause = ulab[i + 1].getUnitName().matches(regexp1) || ulab[i + 1].getUnitName().matches(regexp2);
+					if (iscPause && isnPause) {
+						ulab[i + 1].setStartTime(ulab[i].getStartTime());
+						// System.out.println(ulab[i].getEndTime()+" "+ulab[i].getUnitIndex()+" "+ulab[i].getUnitName());
+						continue;
+					}
+				}
+				if (iscPause) {
+					ulab[i].setUnitName(pauseSymbol);
+				}
+				arrayLabel.add(ulab[i]);
+			}
+
+			// Remove pauses below given threshold
+			for (int i = 0; i < arrayLabel.size(); i++) {
+				UnitLabel ul = arrayLabel.get(i);
+				if (i > 0 && (i + 1) < arrayLabel.size()) {
+					if (ul.getUnitName().equals(pauseSymbol)) {
+						double duration = ul.endTime - ul.startTime;
+						if (!isRealPause(duration)) {
+							System.out.println("deleting... " + ul.startTime + " " + ul.endTime + " " + ul.unitName);
+							UnitLabel pul = arrayLabel.get(i - 1);
+							UnitLabel nul = arrayLabel.get(i + 1);
+							pul.setEndTime(pul.getEndTime() + ((double) duration / 2.0));
+							nul.setStartTime(nul.getStartTime() - ((double) duration / 2.0));
+							arrayLabel.remove(i--);
+						}
+					}
+				}
+			}
+			ulab = arrayLabel.toArray(new UnitLabel[0]);
+
+			// write labels into given file
+			UnitLabel.writeLabFile(ulab, labDir + "/" + basename + labExt);
+		} catch (Exception e) {
+			e.printStackTrace();
+			return false;
+		}
+		return true; // success
+
+	}
+
+	private boolean isRealPause(double phoneDuration) {
+		/*
+		 * TODO: Here we need to modify thresholds
+		 */
+
+		double threshold = Double.parseDouble(getProp(PAUSETHR));
+		if (phoneDuration > (threshold / (double) 1000.0)) {
+			return true;
+		}
+
+		return false;
+	}
+
+	/**
+	 * To get Label Unit DATA (time stamp, index, phone unit)
+	 * 
+	 * @param line
+	 * @return ArrayList contains time stamp, index and phone unit
+	 * @throws IOException
+	 */
+	private ArrayList getLabelUnitData(String line) throws IOException {
+		if (line == null)
+			return null;
+		ArrayList unitData = new ArrayList();
+		StringTokenizer st = new StringTokenizer(line.trim());
+		// the first token is the time
+		unitData.add(st.nextToken());
+		// the second token is the unit index
+		unitData.add(st.nextToken());
+		// the third token is the phone
+		unitData.add(st.nextToken());
+		return unitData;
+	}
+
+	/**
+	 * Provide the progress of computation, in percent, or -1 if that feature is not implemented.
+	 * 
+	 * @return -1 if not implemented, or an integer between 0 and 100.
+	 */
+	public int getProgress() {
+		return progress;
+	}
 
 }
-

marytts-builder/src/main/java/marytts/tools/voiceimport/LabelledFilesInspector.java

@@ -60,325 +60,325 @@ import marytts.util.data.text.ESTTextfileDoubleDataSource;
 import marytts.util.io.FileUtils;
 import marytts.util.string.PrintfFormat;
 
-
 /**
- * For the given texts, compute unit features and align them
- * with the given unit labels.
+ * For the given texts, compute unit features and align them with the given unit labels.
+ * 
  * @author schroed
  *
  */
-public class LabelledFilesInspector extends VoiceImportComponent
-{
-    protected File wavDir;
-    protected File phoneLabDir;
-    protected File pmDir;
+public class LabelledFilesInspector extends VoiceImportComponent {
+	protected File wavDir;
+	protected File phoneLabDir;
+	protected File pmDir;
 
-    protected DatabaseLayout db = null;
-    
-    protected JList fileList;
-    protected JList labels;
-    protected JTextField saveFilename;
-    
-    protected double[] audioSignal;
-    protected AudioFormat audioFormat;
-    protected double[] pitchmarks;
-    protected int samplingRate;
-    protected double tStart;
-    protected double tEnd;
-    private boolean quit = false;
-        
-    protected String extractedDir;
-    protected String extractedWavDir;
-    protected String extractedLabDir;
-    protected String extractedPmDir;
-    
-    public final String PMDIR = "db.pmDir";
-    public final String PMEXT = "db.pmExtension";
- 
-    public String getName(){
-        return "LabelledFilesInspector";
-    }
-    
-    @Override
-    protected void initialiseComp()
-    {
-        extractedDir = db.getProp(db.TEMPDIR);
-        extractedWavDir = extractedDir+"wav/";
-        extractedLabDir = extractedDir+"lab/";
-        extractedPmDir =  extractedDir+"pm/";
-    }
-     
-     public SortedMap getDefaultProps(DatabaseLayout db){
-         this.db = db;
-       if (props == null){
-           props = new TreeMap();
-       }
-       return props;
-   }
-    
-    protected void setupHelp(){         
-        props2Help = new TreeMap();
-    }
+	protected DatabaseLayout db = null;
 
-    public boolean compute() throws IOException
-    {
-        quit = false;
-        wavDir = new File( db.getProp(db.WAVDIR) );
-        if (!wavDir.exists()) throw new IOException("No such directory: "+ wavDir);
-        phoneLabDir = new File( db.getProp(db.LABDIR) );
-        pmDir = new File(db.getProp(PMDIR));
-        
-        File extractedDirFile = new File(extractedDir);
-        if (!extractedDirFile.exists()) extractedDirFile.mkdir();
-        File extractedWavDirFile = new File(extractedWavDir);
-        if (!extractedWavDirFile.exists()) extractedWavDirFile.mkdir();
-        File extractedLabDirFile = new File(extractedLabDir);
-        if (!extractedLabDirFile.exists()) extractedLabDirFile.mkdir();
-        File extractedPmDirFile = new File(extractedPmDir);
-        if (!extractedPmDirFile.exists()) extractedPmDirFile.mkdir();
-        
-        System.out.println( "Proposing for inspection " + bnl.getLength() + " files" );
-        
-        JFrame jf = new JFrame("Inspecting labelled files");
-        jf.addWindowListener(new WindowAdapter() {
-            public void windowClosing(WindowEvent e) {requestQuit();}
-        });
-        Container cont = jf.getContentPane();
-        cont.setLayout(new FlowLayout(FlowLayout.LEFT));
-        fileList = new JList(bnl.getListAsArray());
-        fileList.setSelectionMode(ListSelectionModel.SINGLE_SELECTION);
-        fileList.setVisibleRowCount(-1);
-        JScrollPane fileScroll = new JScrollPane(fileList);
-        fileScroll.setPreferredSize(new Dimension(250, 500));
-        cont.add(fileScroll);
-        fileList.addListSelectionListener(new ListSelectionListener() {
-            public void valueChanged(ListSelectionEvent e)
-            {
-                if (!e.getValueIsAdjusting()) {
-                     String basename = (String) fileList.getSelectedValue();
-                    if (basename != null) {
-                        loadFile(basename);
-                    }
-                }
-            }
-        });
-        
-        labels = new JList();
-        labels.setSelectionMode(ListSelectionModel.SINGLE_INTERVAL_SELECTION);
-        labels.setVisibleRowCount(-1);
-        JScrollPane scroll = new JScrollPane(labels);
-        scroll.setPreferredSize(new Dimension(250, 500));
-        cont.add(scroll);
-        
-        JPanel buttons = new JPanel();
-        cont.add(buttons);
-        buttons.setLayout(new BoxLayout(buttons, BoxLayout.PAGE_AXIS));
+	protected JList fileList;
+	protected JList labels;
+	protected JTextField saveFilename;
 
-        JButton play = new JButton("Play selection");
-        buttons.add(play);
-        play.addActionListener(new ActionListener() {
-            public void actionPerformed(ActionEvent ae)
-            {
-                playCurrentSelection();
-            }
-        });
-        
-        saveFilename = new JTextField(30);
-        buttons.add(saveFilename);
-        
-        JButton save = new JButton("Save");
-        buttons.add(save);
-        save.addActionListener(new ActionListener() {
-            public void actionPerformed(ActionEvent ae)
-            {
-                saveCurrentSelection();
-            }
-        });
-        
-        JButton quitButton = new JButton("Done");
-        buttons.add(quitButton);
-        quitButton.addActionListener(new ActionListener() {
-            public void actionPerformed(ActionEvent ae)
-            {
-                requestQuit();
-            }
-        });
-        
-        jf.pack();
-        jf.setVisible(true);
-        
-        while (!quit) {
-            try {
-                Thread.sleep(100);
-            } catch (InterruptedException ie) {}
-        }
-        jf.setVisible(false);
-        System.out.println("Finished inspecting label files.");
-        return true;
-    }
+	protected double[] audioSignal;
+	protected AudioFormat audioFormat;
+	protected double[] pitchmarks;
+	protected int samplingRate;
+	protected double tStart;
+	protected double tEnd;
+	private boolean quit = false;
 
-    
-    private void loadFile(String basename)
-    {
-        try {
-            File wavFile = new File(wavDir, basename+db.getProp(db.WAVEXT));
-            if (!wavFile.exists()) throw new IllegalArgumentException("File "+wavFile.getAbsolutePath()+" does not exist");
-            File labFile = new File(phoneLabDir, basename+db.getProp(db.LABEXT));
-            if (!labFile.exists()) throw new IllegalArgumentException("File "+labFile.getAbsolutePath()+" does not exist");
-            // pm file is optional
-            File pmFile = new File(pmDir, basename + db.getProp(PMEXT));
-            if (pmFile.exists()) {
-                System.out.println("Loading pitchmarks file "+pmFile.getAbsolutePath());
-                pitchmarks = new ESTTextfileDoubleDataSource(pmFile).getAllData();
-            } else {
-                System.out.println("Pitchmarks file "+pmFile.getAbsolutePath()+" does not exist");
-                pitchmarks = null;
-            }
-            
-            AudioInputStream ais = AudioSystem.getAudioInputStream(wavFile);
-            audioFormat = ais.getFormat();
-            samplingRate = (int) audioFormat.getSampleRate();
-            audioSignal = new AudioDoubleDataSource(ais).getAllData();
-            
-            String file = FileUtils.getFileAsString(labFile, "ASCII");
-            String[] lines = file.split("\n");
-            labels.setListData(lines);
-            
-            saveFilename.setText(basename);
-        } catch (Exception e) {
-            e.printStackTrace();
-        }
-    }
-    
-    private void playCurrentSelection()
-    {
-        if (!delimitSelection()) { // no selection: play entire file
-            tStart = 0;
-            tEnd = audioSignal.length / (double) samplingRate;
-        }
-        // Now we have valid start and end times
-        AudioInputStream playAIS = getSelectedAudio();
-        System.out.println("Playing from "+tStart+" to "+tEnd);
-        try {
-            DataLine.Info clipInfo = new DataLine.Info(Clip.class, audioFormat);
-            Clip clip = (Clip) AudioSystem.getLine(clipInfo);
-            clip.open(playAIS);
-            clip.start();
-        } catch (Exception e) {
-            e.printStackTrace();
-        }
-    }
-    
-    private void saveCurrentSelection()
-    {
-        System.out.println("Saving current selection");
-        if (!delimitSelection()) return;
-        // Now we have valid start and end times
-        File saveWav = new File(extractedWavDir, saveFilename.getText()+db.getProp(db.WAVEXT));
-        File saveLab = new File(extractedLabDir, saveFilename.getText()+db.getProp(db.LABEXT));
-        AudioInputStream selectedAudio = getSelectedAudio();
-        try {
-            // Write audio extract:
-            AudioSystem.write(selectedAudio, AudioFileFormat.Type.WAVE, saveWav);
-            System.out.println("Wrote audio to "+saveWav.getAbsolutePath());
-            // Write label file extract:
-            Object[] selection = labels.getSelectedValues();
-            PrintWriter toLab = new PrintWriter(new FileWriter(saveLab));
-            // these header lines, by the way, serve no discernible purpose here:
-            toLab.println("separator ;");
-            toLab.println("nfields 1");
-            toLab.println("#");
-            for (int i=0; i<selection.length; i++) {
-                String[] parts = ((String)selection[i]).trim().split("\\s+");
-                double time = Double.parseDouble(parts[0]);
-                double newTime = time - tStart;
-                parts[0] = new PrintfFormat(Locale.ENGLISH, "%.5f").sprintf(newTime);
-                for (int j=0; j<parts.length; j++) {
-                    if (j>0) toLab.print("    ");
-                    toLab.print(parts[j]);
-                }
-                toLab.println();
-            }
-            toLab.close();
-            System.out.println("Wrote labels to "+saveLab.getAbsolutePath());
-            // Optionally, write pitchmark extract:
-            if (pitchmarks != null) {
-                int firstpm = -1;
-                int lastPlus1pm = -1;
-                for (int i=0; i<pitchmarks.length; i++) {
-                    if (firstpm == -1) {
-                        if (pitchmarks[i] > tStart) firstpm = i;
-                    } else if (lastPlus1pm == -1) {
-                        if (pitchmarks[i] > tEnd) lastPlus1pm = i;
-                    }
-                }
-                if (lastPlus1pm == -1) lastPlus1pm = pitchmarks.length;
-                float[] pmExtract = new float[lastPlus1pm-firstpm];
-                for (int i=0; i<pmExtract.length; i++) {
-                    pmExtract[i] = (float) (pitchmarks[firstpm+i]-tStart);
-                }
-                String extractedPmFile = extractedPmDir + saveFilename.getText() + db.getProp(PMEXT);
-                new ESTTrackWriter(pmExtract, null, "pitchmarks").doWriteAndClose(extractedPmFile, false, false);
-                System.out.println("Wrote pitchmarks to "+extractedPmFile);
-            }
-        } catch (Exception e) {
-            e.printStackTrace();
-        }
-        
-    }
-    
-    private boolean delimitSelection()
-    {
-        int indices[] = labels.getSelectedIndices();
-        if (indices == null || indices.length == 0) return false;
+	protected String extractedDir;
+	protected String extractedWavDir;
+	protected String extractedLabDir;
+	protected String extractedPmDir;
 
-        if (indices[0] == 0) tStart = 0;
-        else {
-            String prevLine = (String) labels.getModel().getElementAt(indices[0]-1);
-            StringTokenizer t = new StringTokenizer(prevLine);
-            String startTime = t.nextToken();
-            try {
-                tStart = Double.parseDouble(startTime);
-            } catch (NumberFormatException nfe) {
-                tStart = 0;
-            }
-        }
-        String lastLine = (String) labels.getModel().getElementAt(indices[indices.length-1]);
-        StringTokenizer t = new StringTokenizer(lastLine);
-        String endTime = t.nextToken();
-        try {
-            tEnd = Double.parseDouble(endTime);
-        } catch (NumberFormatException nfe) {
-            return false;
-        }
-        return true;
-    }
-    
-    private AudioInputStream getSelectedAudio()
-    {
-        int sStart = (int) (tStart * samplingRate);
-        int sEnd = (int) (tEnd * samplingRate);
-        assert sStart < sEnd;
-        assert sEnd <= audioSignal.length;
-        double[] playSignal = new double[sEnd-sStart];
-        System.arraycopy(audioSignal, sStart, playSignal, 0, sEnd-sStart);
-        return new DDSAudioInputStream(new BufferedDoubleDataSource(playSignal), audioFormat);
-    }
-    
-    private void requestQuit()
-    {
-        quit = true;
-    }
-     
-    /**
-     * Provide the progress of computation, in percent, or -1 if
-     * that feature is not implemented.
-     * @return -1 if not implemented, or an integer between 0 and 100.
-     */
-    public int getProgress()
-    {
-        return -1;
-    }
+	public final String PMDIR = "db.pmDir";
+	public final String PMEXT = "db.pmExtension";
+
+	public String getName() {
+		return "LabelledFilesInspector";
+	}
+
+	@Override
+	protected void initialiseComp() {
+		extractedDir = db.getProp(db.TEMPDIR);
+		extractedWavDir = extractedDir + "wav/";
+		extractedLabDir = extractedDir + "lab/";
+		extractedPmDir = extractedDir + "pm/";
+	}
+
+	public SortedMap getDefaultProps(DatabaseLayout db) {
+		this.db = db;
+		if (props == null) {
+			props = new TreeMap();
+		}
+		return props;
+	}
+
+	protected void setupHelp() {
+		props2Help = new TreeMap();
+	}
+
+	public boolean compute() throws IOException {
+		quit = false;
+		wavDir = new File(db.getProp(db.WAVDIR));
+		if (!wavDir.exists())
+			throw new IOException("No such directory: " + wavDir);
+		phoneLabDir = new File(db.getProp(db.LABDIR));
+		pmDir = new File(db.getProp(PMDIR));
+
+		File extractedDirFile = new File(extractedDir);
+		if (!extractedDirFile.exists())
+			extractedDirFile.mkdir();
+		File extractedWavDirFile = new File(extractedWavDir);
+		if (!extractedWavDirFile.exists())
+			extractedWavDirFile.mkdir();
+		File extractedLabDirFile = new File(extractedLabDir);
+		if (!extractedLabDirFile.exists())
+			extractedLabDirFile.mkdir();
+		File extractedPmDirFile = new File(extractedPmDir);
+		if (!extractedPmDirFile.exists())
+			extractedPmDirFile.mkdir();
+
+		System.out.println("Proposing for inspection " + bnl.getLength() + " files");
+
+		JFrame jf = new JFrame("Inspecting labelled files");
+		jf.addWindowListener(new WindowAdapter() {
+			public void windowClosing(WindowEvent e) {
+				requestQuit();
+			}
+		});
+		Container cont = jf.getContentPane();
+		cont.setLayout(new FlowLayout(FlowLayout.LEFT));
+		fileList = new JList(bnl.getListAsArray());
+		fileList.setSelectionMode(ListSelectionModel.SINGLE_SELECTION);
+		fileList.setVisibleRowCount(-1);
+		JScrollPane fileScroll = new JScrollPane(fileList);
+		fileScroll.setPreferredSize(new Dimension(250, 500));
+		cont.add(fileScroll);
+		fileList.addListSelectionListener(new ListSelectionListener() {
+			public void valueChanged(ListSelectionEvent e) {
+				if (!e.getValueIsAdjusting()) {
+					String basename = (String) fileList.getSelectedValue();
+					if (basename != null) {
+						loadFile(basename);
+					}
+				}
+			}
+		});
+
+		labels = new JList();
+		labels.setSelectionMode(ListSelectionModel.SINGLE_INTERVAL_SELECTION);
+		labels.setVisibleRowCount(-1);
+		JScrollPane scroll = new JScrollPane(labels);
+		scroll.setPreferredSize(new Dimension(250, 500));
+		cont.add(scroll);
+
+		JPanel buttons = new JPanel();
+		cont.add(buttons);
+		buttons.setLayout(new BoxLayout(buttons, BoxLayout.PAGE_AXIS));
+
+		JButton play = new JButton("Play selection");
+		buttons.add(play);
+		play.addActionListener(new ActionListener() {
+			public void actionPerformed(ActionEvent ae) {
+				playCurrentSelection();
+			}
+		});
+
+		saveFilename = new JTextField(30);
+		buttons.add(saveFilename);
+
+		JButton save = new JButton("Save");
+		buttons.add(save);
+		save.addActionListener(new ActionListener() {
+			public void actionPerformed(ActionEvent ae) {
+				saveCurrentSelection();
+			}
+		});
+
+		JButton quitButton = new JButton("Done");
+		buttons.add(quitButton);
+		quitButton.addActionListener(new ActionListener() {
+			public void actionPerformed(ActionEvent ae) {
+				requestQuit();
+			}
+		});
+
+		jf.pack();
+		jf.setVisible(true);
+
+		while (!quit) {
+			try {
+				Thread.sleep(100);
+			} catch (InterruptedException ie) {
+			}
+		}
+		jf.setVisible(false);
+		System.out.println("Finished inspecting label files.");
+		return true;
+	}
+
+	private void loadFile(String basename) {
+		try {
+			File wavFile = new File(wavDir, basename + db.getProp(db.WAVEXT));
+			if (!wavFile.exists())
+				throw new IllegalArgumentException("File " + wavFile.getAbsolutePath() + " does not exist");
+			File labFile = new File(phoneLabDir, basename + db.getProp(db.LABEXT));
+			if (!labFile.exists())
+				throw new IllegalArgumentException("File " + labFile.getAbsolutePath() + " does not exist");
+			// pm file is optional
+			File pmFile = new File(pmDir, basename + db.getProp(PMEXT));
+			if (pmFile.exists()) {
+				System.out.println("Loading pitchmarks file " + pmFile.getAbsolutePath());
+				pitchmarks = new ESTTextfileDoubleDataSource(pmFile).getAllData();
+			} else {
+				System.out.println("Pitchmarks file " + pmFile.getAbsolutePath() + " does not exist");
+				pitchmarks = null;
+			}
+
+			AudioInputStream ais = AudioSystem.getAudioInputStream(wavFile);
+			audioFormat = ais.getFormat();
+			samplingRate = (int) audioFormat.getSampleRate();
+			audioSignal = new AudioDoubleDataSource(ais).getAllData();
+
+			String file = FileUtils.getFileAsString(labFile, "ASCII");
+			String[] lines = file.split("\n");
+			labels.setListData(lines);
+
+			saveFilename.setText(basename);
+		} catch (Exception e) {
+			e.printStackTrace();
+		}
+	}
+
+	private void playCurrentSelection() {
+		if (!delimitSelection()) { // no selection: play entire file
+			tStart = 0;
+			tEnd = audioSignal.length / (double) samplingRate;
+		}
+		// Now we have valid start and end times
+		AudioInputStream playAIS = getSelectedAudio();
+		System.out.println("Playing from " + tStart + " to " + tEnd);
+		try {
+			DataLine.Info clipInfo = new DataLine.Info(Clip.class, audioFormat);
+			Clip clip = (Clip) AudioSystem.getLine(clipInfo);
+			clip.open(playAIS);
+			clip.start();
+		} catch (Exception e) {
+			e.printStackTrace();
+		}
+	}
+
+	private void saveCurrentSelection() {
+		System.out.println("Saving current selection");
+		if (!delimitSelection())
+			return;
+		// Now we have valid start and end times
+		File saveWav = new File(extractedWavDir, saveFilename.getText() + db.getProp(db.WAVEXT));
+		File saveLab = new File(extractedLabDir, saveFilename.getText() + db.getProp(db.LABEXT));
+		AudioInputStream selectedAudio = getSelectedAudio();
+		try {
+			// Write audio extract:
+			AudioSystem.write(selectedAudio, AudioFileFormat.Type.WAVE, saveWav);
+			System.out.println("Wrote audio to " + saveWav.getAbsolutePath());
+			// Write label file extract:
+			Object[] selection = labels.getSelectedValues();
+			PrintWriter toLab = new PrintWriter(new FileWriter(saveLab));
+			// these header lines, by the way, serve no discernible purpose here:
+			toLab.println("separator ;");
+			toLab.println("nfields 1");
+			toLab.println("#");
+			for (int i = 0; i < selection.length; i++) {
+				String[] parts = ((String) selection[i]).trim().split("\\s+");
+				double time = Double.parseDouble(parts[0]);
+				double newTime = time - tStart;
+				parts[0] = new PrintfFormat(Locale.ENGLISH, "%.5f").sprintf(newTime);
+				for (int j = 0; j < parts.length; j++) {
+					if (j > 0)
+						toLab.print("    ");
+					toLab.print(parts[j]);
+				}
+				toLab.println();
+			}
+			toLab.close();
+			System.out.println("Wrote labels to " + saveLab.getAbsolutePath());
+			// Optionally, write pitchmark extract:
+			if (pitchmarks != null) {
+				int firstpm = -1;
+				int lastPlus1pm = -1;
+				for (int i = 0; i < pitchmarks.length; i++) {
+					if (firstpm == -1) {
+						if (pitchmarks[i] > tStart)
+							firstpm = i;
+					} else if (lastPlus1pm == -1) {
+						if (pitchmarks[i] > tEnd)
+							lastPlus1pm = i;
+					}
+				}
+				if (lastPlus1pm == -1)
+					lastPlus1pm = pitchmarks.length;
+				float[] pmExtract = new float[lastPlus1pm - firstpm];
+				for (int i = 0; i < pmExtract.length; i++) {
+					pmExtract[i] = (float) (pitchmarks[firstpm + i] - tStart);
+				}
+				String extractedPmFile = extractedPmDir + saveFilename.getText() + db.getProp(PMEXT);
+				new ESTTrackWriter(pmExtract, null, "pitchmarks").doWriteAndClose(extractedPmFile, false, false);
+				System.out.println("Wrote pitchmarks to " + extractedPmFile);
+			}
+		} catch (Exception e) {
+			e.printStackTrace();
+		}
+
+	}
+
+	private boolean delimitSelection() {
+		int indices[] = labels.getSelectedIndices();
+		if (indices == null || indices.length == 0)
+			return false;
+
+		if (indices[0] == 0)
+			tStart = 0;
+		else {
+			String prevLine = (String) labels.getModel().getElementAt(indices[0] - 1);
+			StringTokenizer t = new StringTokenizer(prevLine);
+			String startTime = t.nextToken();
+			try {
+				tStart = Double.parseDouble(startTime);
+			} catch (NumberFormatException nfe) {
+				tStart = 0;
+			}
+		}
+		String lastLine = (String) labels.getModel().getElementAt(indices[indices.length - 1]);
+		StringTokenizer t = new StringTokenizer(lastLine);
+		String endTime = t.nextToken();
+		try {
+			tEnd = Double.parseDouble(endTime);
+		} catch (NumberFormatException nfe) {
+			return false;
+		}
+		return true;
+	}
+
+	private AudioInputStream getSelectedAudio() {
+		int sStart = (int) (tStart * samplingRate);
+		int sEnd = (int) (tEnd * samplingRate);
+		assert sStart < sEnd;
+		assert sEnd <= audioSignal.length;
+		double[] playSignal = new double[sEnd - sStart];
+		System.arraycopy(audioSignal, sStart, playSignal, 0, sEnd - sStart);
+		return new DDSAudioInputStream(new BufferedDoubleDataSource(playSignal), audioFormat);
+	}
+
+	private void requestQuit() {
+		quit = true;
+	}
+
+	/**
+	 * Provide the progress of computation, in percent, or -1 if that feature is not implemented.
+	 * 
+	 * @return -1 if not implemented, or an integer between 0 and 100.
+	 */
+	public int getProgress() {
+		return -1;
+	}
 
 }
-

marytts-builder/src/main/java/marytts/tools/voiceimport/MCEPMaker.java

@@ -44,225 +44,213 @@ import java.util.TreeMap;
 import marytts.util.data.ESTTrackReader;
 
 public class MCEPMaker extends VoiceImportComponent {
-    
-    protected DatabaseLayout db = null;
-    
-    protected String mcepExt = ".mcep";
-    protected String lpcExt = ".lpc";
-    
-    private final String name = "MCEPMaker";
-    public final String LPCDIR = name + ".lpcDir";
-    public final String MCEPDIR = name + ".mcepDir";
-     
-    public String getName() {
-        return name;
-    }
-    
-   public SortedMap<String, String> getDefaultProps(DatabaseLayout db){
-       this.db = db;
-       if (props == null){
-           props = new TreeMap<String, String>();
-           String rootDir = db.getProp(DatabaseLayout.ROOTDIR);
-           
-           /** Uncomment if you want to use this class for making lpcs
-           props.put(LPCDIR, rootDir
-                   +"lpc"                   
-                   +System.getProperty("file.separator"));
-           **/
-           props.put(MCEPDIR, rootDir
-                   +"mcep"
-                   +System.getProperty("file.separator"));
-       }
-       return props;
-   }
-   
-   protected void setupHelp(){         
-       props2Help = new TreeMap<String, String>();
-       props2Help.put(MCEPDIR, "directory containing the mcep files");
-   }
 
-    /**
-     * Shift the pitchmarks to the closest peak.
-     */
-    private Float[] shiftToClosestPeak( Float[] pmIn, short[] w, int sampleRate ) {
-        
-        final int HORIZON = 32; // <= number of samples to seek before and after the pitchmark
-        Float[] pmOut = new Float[pmIn.length];
-        
-        /* Browse the pitchmarks */
-        int pm = 0;
-        int pmwmax = w.length - 1;
-        int TO = 0;
-        int max = 0;
-        for ( int pi = 0; pi < pmIn.length; pi++ ) {
-            pm = (int)( pmIn[pi].floatValue() * sampleRate );
-            // If the pitchmark goes out of the waveform (this sometimes
-            // happens with the last one due to rounding errors), just clip it.
-            if ( pm > pmwmax ) {
-                // If this was not the last pitchmark, there is a problem
-                if ( pi < (pmIn.length-1)) {
-                    throw new RuntimeException( "Some pitchmarks are located above the location of the last waveform sample !" );
-                }
-                // Else, if it was the last pitchmark, clip it:
-                pmOut[pi] = new Float( (double)(pmwmax) / (double)(sampleRate) );
-            }
-            // Else, if the pitchmark is in the waveform:
-            else {
-                /* Seek the max of the wav samples around the pitchmark */
-                max = pm;
-                // - Back:
-                TO = (pm-HORIZON) < 0 ? 0 : (pm-HORIZON);
-                for ( int i = pm-1; i >= TO; i-- ) {
-                    if ( w[i] > w[max] ) max = i;
-                }
-                // - Forth:
-                TO = (pm+HORIZON+1) > w.length ? w.length : (pm+HORIZON+1);
-                for ( int i = pm+1; i < TO; i++ ) {
-                    if ( w[i] >= w[max] ) max = i;
-                }
-                /* Translate the pitchmark */
-                pmOut[pi] = new Float( (double)(max) / (double)(sampleRate) );
-            }
-        }
-        
-        return( pmOut );
-    }
-    
-    /**
-     * Shift the pitchmarks to the previous zero crossing.
-     */
-    private Float[] shiftToPreviousZero( Float[] pmIn, short[] w, int sampleRate ) {
-        
-        final int HORIZON = 32; // <= number of samples to seek before the pitchmark
-        Float[] pmOut = new Float[pmIn.length];
-        
-        /* Browse the pitchmarks */
-        int pm = 0;
-        int TO = 0;
-        int zero = 0;
-        for ( int pi = 0; pi < pmIn.length; pi++ ) {
-            pm = (int)( pmIn[pi].floatValue() * sampleRate );
-            /* If the initial pitchmark is on a zero, don't shift the pitchmark. */
-            if ( w[pm] == 0 ) {
-                pmOut[pi] = pmIn[pi];
-                continue;
-            }
-            /* Else: */
-            /* Seek the zero crossing preceding the pitchmark */
-            TO = (pm-HORIZON) < 0 ? 0 : (pm-HORIZON);
-            for ( zero = (pm-1); ( zero > TO ) && ( (w[zero]*w[zero+1]) > 0 ); zero-- );
-            /* If no zero crossing was found, don't move the pitchmark */
-            if ( (zero == TO) && ( (w[zero]*w[zero+1]) > 0 )  ) {
-                pmOut[pi] = pmIn[pi];
-            }
-            /* If a zero crossing was found, translate the pitchmark */
-            else {
-                pmOut[pi] = new Float( (double)( (-w[zero]) <  w[zero+1] ?  zero : (zero+1) ) / (double)(sampleRate) );
-            }
-        }
-        
-        return( pmOut );
-    }
-    
-    /**
-     * Rectification of the pitchmarks.
-     */
-    private void tweakThePitchmarks( String[] baseNameArray ) throws IOException {
-        
-        System.out.println("---- Correcting the pitchmarks..." );
-        
-        /* Ensure the existence of the target directory */
-        File dir = new File( db.getProp(DatabaseLayout.PMDIR));
-        if (!dir.exists()) { 
-            System.out.println( "Creating the directory [" + db.getProp(DatabaseLayout.PMDIR) + "]." );
-            dir.mkdir();
-        }
-        
-        // For each file
-        for ( int f = 0; f < baseNameArray.length; f++ ) {
-        //for ( int f = 0; f < 1; f++ ) {
-            /* Load the pitchmark file */
-            //System.out.println( baseNameArray[f] );
-            String fName = db.getProp(DatabaseLayout.PMDIR) + baseNameArray[f] + db.getProp(DatabaseLayout.PMEXT);
-            ESTTrackReader pmFileIn = new ESTTrackReader( fName );
-            /* Wrap the primitive floats so that we can use vectors thereafter */
-            float[] pmInPrimitive = pmFileIn.getTimes();
-            Float[] pmIn = new Float[pmInPrimitive.length];
-            for ( int i = 0; i < pmInPrimitive.length; i++ ) {
-                pmIn[i] = (Float)Array.get( pmInPrimitive, i );
-            }
-            /* Load the wav file */
-            fName = db.getProp(DatabaseLayout.WAVDIR) + baseNameArray[f] + db.getProp(DatabaseLayout.WAVEXT);
-            WavReader wf = new WavReader( fName );
-            short[] w = wf.getSamples();
-            Float[] pmOut = null;
-            try {
-                /* Shift to the closest peak */
-                pmOut = shiftToClosestPeak( pmIn, w, wf.getSampleRate() );
-                /* Shift to the zero immediately preceding the closest peak */
-                pmOut = shiftToPreviousZero( pmOut, w, wf.getSampleRate() );
-            }
-            catch ( RuntimeException e ) {
-                throw new RuntimeException( "For utterance [" + baseNameArray[f] + "]:" , e );
-            }
-            /* Export the corrected pitchmarks as an EST file */
-            fName = db.getProp(DatabaseLayout.PMDIR) + baseNameArray[f] + db.getProp(DatabaseLayout.PMEXT);
-            DataOutputStream dos = null;
-            try {
-                dos = new DataOutputStream( new BufferedOutputStream( new FileOutputStream( fName ) ) );
-            }
-            catch ( FileNotFoundException e ) {
-                throw new RuntimeException( "Can't open file [" + fName + "] for writing.", e );
-            }
-            dos.writeBytes( "EST_File Track\n" +
-                    "DataType ascii\n" +
-                    "NumFrames " + Integer.toString(pmOut.length) + "\n" +
-                    "NumChannels 0\n" +
-                    "NumAuxChannels 0\n" +
-                    "EqualSpace 0\n" +
-                    "BreaksPresent true\n" +
-                    "EST_Header_End\n" );
-            for ( int i = 0; i < pmOut.length; i++ ) {
-                dos.writeBytes( pmOut[i].toString() );
-                dos.writeBytes( "\t1\n" );
-            }
-            dos.flush();
-            dos.close();
-        }
-        
-    }
-    
-    /**
-     * The standard compute() method of the VoiceImportComponent interface.
-     */
-    public boolean compute() throws IOException {
-        
-        // do not attempt to launch ESTCaller if bnl is empty
-        if (bnl.getLength() < 1) {
-            logger.error("BaseNameList is empty, cannot proceed!");
-            return false;
-        }
-        
-        String[] baseNameArray = bnl.getListAsArray();
-        System.out.println( "Computing Mel cepstra for [" + baseNameArray.length + "] utterances." );
-        ESTCaller caller = new ESTCaller(db);
-        caller.make_mcep( baseNameArray, 
-                db.getProp(DatabaseLayout.PMDIR),
-                db.getProp(DatabaseLayout.PMEXT),
-                getProp(MCEPDIR),
-                mcepExt);
-        
-        return( true );
-    }
-    
-    /**
-     * Provide the progress of computation, in percent, or -1 if
-     * that feature is not implemented.
-     * @return -1 if not implemented, or an integer between 0 and 100.
-     */
-    public int getProgress()
-    {
-        return -1;
-    }
+	protected DatabaseLayout db = null;
+
+	protected String mcepExt = ".mcep";
+	protected String lpcExt = ".lpc";
+
+	private final String name = "MCEPMaker";
+	public final String LPCDIR = name + ".lpcDir";
+	public final String MCEPDIR = name + ".mcepDir";
+
+	public String getName() {
+		return name;
+	}
+
+	public SortedMap<String, String> getDefaultProps(DatabaseLayout db) {
+		this.db = db;
+		if (props == null) {
+			props = new TreeMap<String, String>();
+			String rootDir = db.getProp(DatabaseLayout.ROOTDIR);
+
+			/**
+			 * Uncomment if you want to use this class for making lpcs props.put(LPCDIR, rootDir +"lpc"
+			 * +System.getProperty("file.separator"));
+			 **/
+			props.put(MCEPDIR, rootDir + "mcep" + System.getProperty("file.separator"));
+		}
+		return props;
+	}
+
+	protected void setupHelp() {
+		props2Help = new TreeMap<String, String>();
+		props2Help.put(MCEPDIR, "directory containing the mcep files");
+	}
+
+	/**
+	 * Shift the pitchmarks to the closest peak.
+	 */
+	private Float[] shiftToClosestPeak(Float[] pmIn, short[] w, int sampleRate) {
+
+		final int HORIZON = 32; // <= number of samples to seek before and after the pitchmark
+		Float[] pmOut = new Float[pmIn.length];
+
+		/* Browse the pitchmarks */
+		int pm = 0;
+		int pmwmax = w.length - 1;
+		int TO = 0;
+		int max = 0;
+		for (int pi = 0; pi < pmIn.length; pi++) {
+			pm = (int) (pmIn[pi].floatValue() * sampleRate);
+			// If the pitchmark goes out of the waveform (this sometimes
+			// happens with the last one due to rounding errors), just clip it.
+			if (pm > pmwmax) {
+				// If this was not the last pitchmark, there is a problem
+				if (pi < (pmIn.length - 1)) {
+					throw new RuntimeException("Some pitchmarks are located above the location of the last waveform sample !");
+				}
+				// Else, if it was the last pitchmark, clip it:
+				pmOut[pi] = new Float((double) (pmwmax) / (double) (sampleRate));
+			}
+			// Else, if the pitchmark is in the waveform:
+			else {
+				/* Seek the max of the wav samples around the pitchmark */
+				max = pm;
+				// - Back:
+				TO = (pm - HORIZON) < 0 ? 0 : (pm - HORIZON);
+				for (int i = pm - 1; i >= TO; i--) {
+					if (w[i] > w[max])
+						max = i;
+				}
+				// - Forth:
+				TO = (pm + HORIZON + 1) > w.length ? w.length : (pm + HORIZON + 1);
+				for (int i = pm + 1; i < TO; i++) {
+					if (w[i] >= w[max])
+						max = i;
+				}
+				/* Translate the pitchmark */
+				pmOut[pi] = new Float((double) (max) / (double) (sampleRate));
+			}
+		}
+
+		return (pmOut);
+	}
+
+	/**
+	 * Shift the pitchmarks to the previous zero crossing.
+	 */
+	private Float[] shiftToPreviousZero(Float[] pmIn, short[] w, int sampleRate) {
+
+		final int HORIZON = 32; // <= number of samples to seek before the pitchmark
+		Float[] pmOut = new Float[pmIn.length];
+
+		/* Browse the pitchmarks */
+		int pm = 0;
+		int TO = 0;
+		int zero = 0;
+		for (int pi = 0; pi < pmIn.length; pi++) {
+			pm = (int) (pmIn[pi].floatValue() * sampleRate);
+			/* If the initial pitchmark is on a zero, don't shift the pitchmark. */
+			if (w[pm] == 0) {
+				pmOut[pi] = pmIn[pi];
+				continue;
+			}
+			/* Else: */
+			/* Seek the zero crossing preceding the pitchmark */
+			TO = (pm - HORIZON) < 0 ? 0 : (pm - HORIZON);
+			for (zero = (pm - 1); (zero > TO) && ((w[zero] * w[zero + 1]) > 0); zero--)
+				;
+			/* If no zero crossing was found, don't move the pitchmark */
+			if ((zero == TO) && ((w[zero] * w[zero + 1]) > 0)) {
+				pmOut[pi] = pmIn[pi];
+			}
+			/* If a zero crossing was found, translate the pitchmark */
+			else {
+				pmOut[pi] = new Float((double) ((-w[zero]) < w[zero + 1] ? zero : (zero + 1)) / (double) (sampleRate));
+			}
+		}
+
+		return (pmOut);
+	}
+
+	/**
+	 * Rectification of the pitchmarks.
+	 */
+	private void tweakThePitchmarks(String[] baseNameArray) throws IOException {
+
+		System.out.println("---- Correcting the pitchmarks...");
+
+		/* Ensure the existence of the target directory */
+		File dir = new File(db.getProp(DatabaseLayout.PMDIR));
+		if (!dir.exists()) {
+			System.out.println("Creating the directory [" + db.getProp(DatabaseLayout.PMDIR) + "].");
+			dir.mkdir();
+		}
+
+		// For each file
+		for (int f = 0; f < baseNameArray.length; f++) {
+			// for ( int f = 0; f < 1; f++ ) {
+			/* Load the pitchmark file */
+			// System.out.println( baseNameArray[f] );
+			String fName = db.getProp(DatabaseLayout.PMDIR) + baseNameArray[f] + db.getProp(DatabaseLayout.PMEXT);
+			ESTTrackReader pmFileIn = new ESTTrackReader(fName);
+			/* Wrap the primitive floats so that we can use vectors thereafter */
+			float[] pmInPrimitive = pmFileIn.getTimes();
+			Float[] pmIn = new Float[pmInPrimitive.length];
+			for (int i = 0; i < pmInPrimitive.length; i++) {
+				pmIn[i] = (Float) Array.get(pmInPrimitive, i);
+			}
+			/* Load the wav file */
+			fName = db.getProp(DatabaseLayout.WAVDIR) + baseNameArray[f] + db.getProp(DatabaseLayout.WAVEXT);
+			WavReader wf = new WavReader(fName);
+			short[] w = wf.getSamples();
+			Float[] pmOut = null;
+			try {
+				/* Shift to the closest peak */
+				pmOut = shiftToClosestPeak(pmIn, w, wf.getSampleRate());
+				/* Shift to the zero immediately preceding the closest peak */
+				pmOut = shiftToPreviousZero(pmOut, w, wf.getSampleRate());
+			} catch (RuntimeException e) {
+				throw new RuntimeException("For utterance [" + baseNameArray[f] + "]:", e);
+			}
+			/* Export the corrected pitchmarks as an EST file */
+			fName = db.getProp(DatabaseLayout.PMDIR) + baseNameArray[f] + db.getProp(DatabaseLayout.PMEXT);
+			DataOutputStream dos = null;
+			try {
+				dos = new DataOutputStream(new BufferedOutputStream(new FileOutputStream(fName)));
+			} catch (FileNotFoundException e) {
+				throw new RuntimeException("Can't open file [" + fName + "] for writing.", e);
+			}
+			dos.writeBytes("EST_File Track\n" + "DataType ascii\n" + "NumFrames " + Integer.toString(pmOut.length) + "\n"
+					+ "NumChannels 0\n" + "NumAuxChannels 0\n" + "EqualSpace 0\n" + "BreaksPresent true\n" + "EST_Header_End\n");
+			for (int i = 0; i < pmOut.length; i++) {
+				dos.writeBytes(pmOut[i].toString());
+				dos.writeBytes("\t1\n");
+			}
+			dos.flush();
+			dos.close();
+		}
+
+	}
+
+	/**
+	 * The standard compute() method of the VoiceImportComponent interface.
+	 */
+	public boolean compute() throws IOException {
+
+		// do not attempt to launch ESTCaller if bnl is empty
+		if (bnl.getLength() < 1) {
+			logger.error("BaseNameList is empty, cannot proceed!");
+			return false;
+		}
+
+		String[] baseNameArray = bnl.getListAsArray();
+		System.out.println("Computing Mel cepstra for [" + baseNameArray.length + "] utterances.");
+		ESTCaller caller = new ESTCaller(db);
+		caller.make_mcep(baseNameArray, db.getProp(DatabaseLayout.PMDIR), db.getProp(DatabaseLayout.PMEXT), getProp(MCEPDIR),
+				mcepExt);
+
+		return (true);
+	}
+
+	/**
+	 * Provide the progress of computation, in percent, or -1 if that feature is not implemented.
+	 * 
+	 * @return -1 if not implemented, or an integer between 0 and 100.
+	 */
+	public int getProgress() {
+		return -1;
+	}
 
 }

marytts-builder/src/main/java/marytts/tools/voiceimport/MCepTimelineMaker.java

@@ -42,230 +42,222 @@ import java.util.TreeMap;
 import marytts.unitselection.data.MCepDatagram;
 import marytts.util.data.ESTTrackReader;
 
-
 /**
- * The mcepTimelineMaker class takes a database root directory and a list of basenames,
- * and converts the related wav files into a mcep timeline in Mary format.
+ * The mcepTimelineMaker class takes a database root directory and a list of basenames, and converts the related wav files into a
+ * mcep timeline in Mary format.
  * 
  * @author sacha
  */
-public class MCepTimelineMaker extends VoiceImportComponent
-{ 
+public class MCepTimelineMaker extends VoiceImportComponent {
 
-    protected DatabaseLayout db = null;
-    protected int percent = 0;
-    protected String mcepExt = ".mcep";
-    
-    public final String MCEPDIR = "MCepTimelineMaker.mcepDir";
-    public final String MCEPTIMELINE = "MCepTimelineMaker.mcepTimeline";
-    
-    public String getName(){
-        return "MCepTimelineMaker";
-    }
-    
-    
-    public SortedMap<String,String> getDefaultProps(DatabaseLayout theDb){
-        this.db = theDb;
-        if (props == null){
-            props = new TreeMap<String, String>();
-            props.put(MCEPDIR, db.getProp(db.ROOTDIR)
-                    +"mcep"
-                    +System.getProperty("file.separator"));
-            props.put(MCEPTIMELINE, db.getProp(db.FILEDIR)
-                    +"timeline_mcep"+db.getProp(db.MARYEXT));  
-        }
-        return props;
-    }
-    
-    protected void setupHelp(){         
-        props2Help = new TreeMap<String, String>();
-        props2Help.put(MCEPDIR, "directory containing the mcep files");
-        props2Help.put(MCEPTIMELINE,"file containing all mcep files. Will be created by this module");  
-    }
-    
-    /**
-     *  Reads and concatenates a list of mcep EST tracks into one single timeline file.
-     *
-     */
-    public boolean compute()
-    {
-        System.out.println("---- Importing Mel Cepstrum coefficients\n\n");
-        System.out.println("Base directory: " + db.getProp(db.ROOTDIR) + "\n");
-        
-        /* Export the basename list into an array of strings */
-        String[] baseNameArray = bnl.getListAsArray();
-        
-        /* Prepare the output directory for the timelines if it does not exist */
-        File timelineDir = new File(db.getProp(db.FILEDIR));
-        
-        
-        try{
-            /* 1) Determine the reference sampling rate as being the sample rate of the first encountered
-             *    wav file */
-            WavReader wav = new WavReader(db.getProp(db.WAVDIR) 
-                    + baseNameArray[0] + db.getProp(db.WAVEXT));
-            int globSampleRate = wav.getSampleRate();
-            System.out.println("---- Detected a global sample rate of: [" + globSampleRate + "] Hz." );
-            
-            /* 2) Scan all the EST Mel Cepstrum Track files for min, max and total timeline duration */
-            
-            System.out.println("---- Scanning for MCep min and MCep max..." );
-            
-            ESTTrackReader mcepFile;    // Structure that holds the mcep track data
-            float[] current;             // local [min,max] vector for the current mcep track file
-            float mcepMin, mcepMax, mcepRange;       // Global min/max/range values for the mcep coefficients
-            float totalDuration = 0.0f;  // Accumulator for the total timeline duration
-            long numDatagrams = 0l; // Total number of mcep datagrams in the timeline file
-            int numMCep = 0;              // Number of mcep channels, assumed from the first mcep file
-            
-            /* Initialize with the first file: */
-            /* - open and load */
-            // System.out.println( baseNameArray[0] );
-            File first = new File(getProp(MCEPDIR) + baseNameArray[0] + mcepExt);
-            if (!first.canRead()) {
-                throw new IOException("File " + first.getAbsolutePath() + " not readable!");
-            }
-            mcepFile = new ESTTrackReader(first.getAbsolutePath());
-            /* - get the min and the max */
-            current = mcepFile.getMinMax();
-            mcepMin = current[0];
-            mcepMax = current[1];
-            /* - accumulate the file duration */
-            totalDuration += mcepFile.getTimeSpan();
-            /* - accumulate the number of datagrams: */
-            numDatagrams += mcepFile.getNumFrames();
-            /* - get the number of mcep channels: */
-            numMCep = mcepFile.getNumChannels();
-            System.out.println("Assuming that the number of Mel Cepstrum coefficients is: [" + numMCep + "] coefficients." );
-            
-            /* Then, browse the remaining files: */
-            for ( int i = 1; i < baseNameArray.length; i++ ) {
-                percent = 50*i/baseNameArray.length;
-                /* - open+load */
-                // System.out.println( baseNameArray[i] );
-                mcepFile = new ESTTrackReader(getProp(MCEPDIR) 
-                        + baseNameArray[i] + mcepExt);
-                /* - get min and max */
-                current = mcepFile.getMinMax();
-                if ( current[0] < mcepMin ) { mcepMin = current[0]; }
-                if ( current[1] > mcepMax ) { mcepMax = current[1]; }
-                /* - accumulate and approximate of the total speech duration (to build the index) */
-                totalDuration += mcepFile.getTimeSpan();
-                /* - accumulate the number of datagrams: */
-                numDatagrams += mcepFile.getNumFrames();
-            }
-            mcepRange = mcepMax - mcepMin;
-            /* NOTE: accumulating the total mcep timeline duration (which is necessary for dimensioning the index)
-             * from the mcep track times is slightly more imprecise than accumulating durations from the residuals,
-             * but it avoids another loop through on-disk files. */
-            
-            System.out.println("mcepMin   = " + mcepMin );
-            System.out.println("mcepMax   = " + mcepMax );
-            System.out.println("mcepRange = " + mcepRange );
-            
-            System.out.println("---- Done." );
-            
-            System.out.println("---- Translating the EST MCep tracks..." );
-            
-            /* 3) Open the destination timeline file */
-            
-            /* Make the file name */
-            System.out.println( "Will create the mcep timeline in file [" 
-                    + getProp(MCEPTIMELINE) + "]." );
-            
-            /* An example of processing header: */
-            Properties props = new Properties();
-            String hdrCmdLine = "\n$ESTDIR/bin/sig2fv "
-                + "-window_type hamming -factor 2.5 -otype est_binary -coefs melcep -melcep_order 12 -fbank_order 24 -shift 0.01 -preemph 0.97 "
-                + "-pm PITCHMARKFILE.pm -o melcepDir/mcepFile.mcep WAVDIR/WAVFILE.wav\n";
-            props.setProperty("command", hdrCmdLine);
-            props.setProperty("mcep.order", String.valueOf(numMCep));
-            props.setProperty("mcep.min", String.valueOf(mcepMin));
-            props.setProperty("mcep.range", String.valueOf(mcepRange));
-            ByteArrayOutputStream baos = new ByteArrayOutputStream();
-            props.store(baos, null);
-            String processingHeader = baos.toString("latin1");
-            
-            
-            /* Instantiate the TimelineWriter: */
-            TimelineWriter mcepTimeline = new TimelineWriter( getProp(MCEPTIMELINE), processingHeader, globSampleRate, 0.1 );
-            
-            
-            /* 4) Write the datagrams and feed the index */
-            
-            long totalTime = 0l;
-            
-            /* For each EST track file: */
-            for ( int i = 0; i < baseNameArray.length; i++ ) {
-                percent = 50+50*i/baseNameArray.length;
-                /* - open+load */
-                System.out.println( baseNameArray[i] );
-                mcepFile = new ESTTrackReader( getProp(MCEPDIR) 
-                        + baseNameArray[i] + mcepExt);
-                wav = new WavReader(db.getProp(db.WAVDIR) 
-                        + baseNameArray[i] + db.getProp(db.WAVEXT));
-                /* - For each frame in the mcep file: */
-                int frameStart = 0;
-                int frameEnd = 0;
-                int duration = 0;
-                long localTime = 0l;
-                for ( int f = 0; f < mcepFile.getNumFrames(); f++ ) {
-                    /* Get the datagram duration */
-                    frameStart = frameEnd;
-                    frameEnd = (int)( (double)mcepFile.getTime( f ) * (double)(globSampleRate) );
-                    duration = frameEnd - frameStart;
-                    /* NOTE: quantization is no more performed below, code&comments kept for archiving. */
-                    /* Quantize the mcep coeffs: */
-                    // short[] quantizedFrame = General.quantize( mcepFile.getFrame( f ), mcepMin, mcepRange );
-                    /* Make a datagram from the quantized mcep coefficients: */
-                    /* for ( int k = 0; k < quantizedFrame.length; k++ ) {
-                        datagramContents.writeShort( quantizedFrame[k] );
-                    } */
-                    /* Feed the datagram to the timeline */
-                    mcepTimeline.feed( new MCepDatagram( duration, mcepFile.getFrame( f ) ) , globSampleRate );
-                    totalTime += duration;
-                    localTime += duration;
-                }
-                // System.out.println( baseNameArray[i] + " -> mcep file says [" + localTime + "] samples, wav file says ["+ wav.getNumSamples() + "] samples." );
-                
-            }
-            mcepTimeline.close();
-            
-            System.out.println("---- Done." );
-            
-            /* 7) Print some stats and close the file */
-            System.out.println( "---- mcep timeline result:");
-            System.out.println( "Number of files scanned: " + baseNameArray.length );
-            System.out.println( "Total speech duration: [" + totalTime + "] samples / [" + ((double)(totalTime) / (double)(globSampleRate)) + "] seconds." );
-            System.out.println( "(Speech duration approximated from EST Track float times: [" + totalDuration + "] seconds.)" );
-            System.out.println( "Number of frames: [" + numDatagrams + "]." );
-            System.out.println( "Size of the index: [" + mcepTimeline.getIndex().getNumIdx() + "] ("
-                    + (mcepTimeline.getIndex().getNumIdx() * 16) + " bytes, i.e. "
-                    + new DecimalFormat("#.##").format((double)(mcepTimeline.getIndex().getNumIdx()) * 16.0 / 1048576.0) + " megs).");
-            System.out.println( "---- mcep timeline done.");
-            
-        }
-        catch ( SecurityException e ) {
-            System.err.println( "Error: you don't have write access to the target database directory." );
-            return false;
-        }
-        catch (Exception e) {
-            e.printStackTrace();
-            System.err.println(e);
-            return false;
-        }
-        
-        return( true );
-    }
-    
-    /**
-     * Provide the progress of computation, in percent, or -1 if
-     * that feature is not implemented.
-     * @return -1 if not implemented, or an integer between 0 and 100.
-     */
-    public int getProgress()
-    {
-        return percent;
-    }
+	protected DatabaseLayout db = null;
+	protected int percent = 0;
+	protected String mcepExt = ".mcep";
+
+	public final String MCEPDIR = "MCepTimelineMaker.mcepDir";
+	public final String MCEPTIMELINE = "MCepTimelineMaker.mcepTimeline";
+
+	public String getName() {
+		return "MCepTimelineMaker";
+	}
+
+	public SortedMap<String, String> getDefaultProps(DatabaseLayout theDb) {
+		this.db = theDb;
+		if (props == null) {
+			props = new TreeMap<String, String>();
+			props.put(MCEPDIR, db.getProp(db.ROOTDIR) + "mcep" + System.getProperty("file.separator"));
+			props.put(MCEPTIMELINE, db.getProp(db.FILEDIR) + "timeline_mcep" + db.getProp(db.MARYEXT));
+		}
+		return props;
+	}
+
+	protected void setupHelp() {
+		props2Help = new TreeMap<String, String>();
+		props2Help.put(MCEPDIR, "directory containing the mcep files");
+		props2Help.put(MCEPTIMELINE, "file containing all mcep files. Will be created by this module");
+	}
+
+	/**
+	 * Reads and concatenates a list of mcep EST tracks into one single timeline file.
+	 *
+	 */
+	public boolean compute() {
+		System.out.println("---- Importing Mel Cepstrum coefficients\n\n");
+		System.out.println("Base directory: " + db.getProp(db.ROOTDIR) + "\n");
+
+		/* Export the basename list into an array of strings */
+		String[] baseNameArray = bnl.getListAsArray();
+
+		/* Prepare the output directory for the timelines if it does not exist */
+		File timelineDir = new File(db.getProp(db.FILEDIR));
+
+		try {
+			/*
+			 * 1) Determine the reference sampling rate as being the sample rate of the first encountered wav file
+			 */
+			WavReader wav = new WavReader(db.getProp(db.WAVDIR) + baseNameArray[0] + db.getProp(db.WAVEXT));
+			int globSampleRate = wav.getSampleRate();
+			System.out.println("---- Detected a global sample rate of: [" + globSampleRate + "] Hz.");
+
+			/* 2) Scan all the EST Mel Cepstrum Track files for min, max and total timeline duration */
+
+			System.out.println("---- Scanning for MCep min and MCep max...");
+
+			ESTTrackReader mcepFile; // Structure that holds the mcep track data
+			float[] current; // local [min,max] vector for the current mcep track file
+			float mcepMin, mcepMax, mcepRange; // Global min/max/range values for the mcep coefficients
+			float totalDuration = 0.0f; // Accumulator for the total timeline duration
+			long numDatagrams = 0l; // Total number of mcep datagrams in the timeline file
+			int numMCep = 0; // Number of mcep channels, assumed from the first mcep file
+
+			/* Initialize with the first file: */
+			/* - open and load */
+			// System.out.println( baseNameArray[0] );
+			File first = new File(getProp(MCEPDIR) + baseNameArray[0] + mcepExt);
+			if (!first.canRead()) {
+				throw new IOException("File " + first.getAbsolutePath() + " not readable!");
+			}
+			mcepFile = new ESTTrackReader(first.getAbsolutePath());
+			/* - get the min and the max */
+			current = mcepFile.getMinMax();
+			mcepMin = current[0];
+			mcepMax = current[1];
+			/* - accumulate the file duration */
+			totalDuration += mcepFile.getTimeSpan();
+			/* - accumulate the number of datagrams: */
+			numDatagrams += mcepFile.getNumFrames();
+			/* - get the number of mcep channels: */
+			numMCep = mcepFile.getNumChannels();
+			System.out.println("Assuming that the number of Mel Cepstrum coefficients is: [" + numMCep + "] coefficients.");
+
+			/* Then, browse the remaining files: */
+			for (int i = 1; i < baseNameArray.length; i++) {
+				percent = 50 * i / baseNameArray.length;
+				/* - open+load */
+				// System.out.println( baseNameArray[i] );
+				mcepFile = new ESTTrackReader(getProp(MCEPDIR) + baseNameArray[i] + mcepExt);
+				/* - get min and max */
+				current = mcepFile.getMinMax();
+				if (current[0] < mcepMin) {
+					mcepMin = current[0];
+				}
+				if (current[1] > mcepMax) {
+					mcepMax = current[1];
+				}
+				/* - accumulate and approximate of the total speech duration (to build the index) */
+				totalDuration += mcepFile.getTimeSpan();
+				/* - accumulate the number of datagrams: */
+				numDatagrams += mcepFile.getNumFrames();
+			}
+			mcepRange = mcepMax - mcepMin;
+			/*
+			 * NOTE: accumulating the total mcep timeline duration (which is necessary for dimensioning the index) from the mcep
+			 * track times is slightly more imprecise than accumulating durations from the residuals, but it avoids another loop
+			 * through on-disk files.
+			 */
+
+			System.out.println("mcepMin   = " + mcepMin);
+			System.out.println("mcepMax   = " + mcepMax);
+			System.out.println("mcepRange = " + mcepRange);
+
+			System.out.println("---- Done.");
+
+			System.out.println("---- Translating the EST MCep tracks...");
+
+			/* 3) Open the destination timeline file */
+
+			/* Make the file name */
+			System.out.println("Will create the mcep timeline in file [" + getProp(MCEPTIMELINE) + "].");
+
+			/* An example of processing header: */
+			Properties props = new Properties();
+			String hdrCmdLine = "\n$ESTDIR/bin/sig2fv "
+					+ "-window_type hamming -factor 2.5 -otype est_binary -coefs melcep -melcep_order 12 -fbank_order 24 -shift 0.01 -preemph 0.97 "
+					+ "-pm PITCHMARKFILE.pm -o melcepDir/mcepFile.mcep WAVDIR/WAVFILE.wav\n";
+			props.setProperty("command", hdrCmdLine);
+			props.setProperty("mcep.order", String.valueOf(numMCep));
+			props.setProperty("mcep.min", String.valueOf(mcepMin));
+			props.setProperty("mcep.range", String.valueOf(mcepRange));
+			ByteArrayOutputStream baos = new ByteArrayOutputStream();
+			props.store(baos, null);
+			String processingHeader = baos.toString("latin1");
+
+			/* Instantiate the TimelineWriter: */
+			TimelineWriter mcepTimeline = new TimelineWriter(getProp(MCEPTIMELINE), processingHeader, globSampleRate, 0.1);
+
+			/* 4) Write the datagrams and feed the index */
+
+			long totalTime = 0l;
+
+			/* For each EST track file: */
+			for (int i = 0; i < baseNameArray.length; i++) {
+				percent = 50 + 50 * i / baseNameArray.length;
+				/* - open+load */
+				System.out.println(baseNameArray[i]);
+				mcepFile = new ESTTrackReader(getProp(MCEPDIR) + baseNameArray[i] + mcepExt);
+				wav = new WavReader(db.getProp(db.WAVDIR) + baseNameArray[i] + db.getProp(db.WAVEXT));
+				/* - For each frame in the mcep file: */
+				int frameStart = 0;
+				int frameEnd = 0;
+				int duration = 0;
+				long localTime = 0l;
+				for (int f = 0; f < mcepFile.getNumFrames(); f++) {
+					/* Get the datagram duration */
+					frameStart = frameEnd;
+					frameEnd = (int) ((double) mcepFile.getTime(f) * (double) (globSampleRate));
+					duration = frameEnd - frameStart;
+					/* NOTE: quantization is no more performed below, code&comments kept for archiving. */
+					/* Quantize the mcep coeffs: */
+					// short[] quantizedFrame = General.quantize( mcepFile.getFrame( f ), mcepMin, mcepRange );
+					/* Make a datagram from the quantized mcep coefficients: */
+					/*
+					 * for ( int k = 0; k < quantizedFrame.length; k++ ) { datagramContents.writeShort( quantizedFrame[k] ); }
+					 */
+					/* Feed the datagram to the timeline */
+					mcepTimeline.feed(new MCepDatagram(duration, mcepFile.getFrame(f)), globSampleRate);
+					totalTime += duration;
+					localTime += duration;
+				}
+				// System.out.println( baseNameArray[i] + " -> mcep file says [" + localTime + "] samples, wav file says ["+
+				// wav.getNumSamples() + "] samples." );
+
+			}
+			mcepTimeline.close();
+
+			System.out.println("---- Done.");
+
+			/* 7) Print some stats and close the file */
+			System.out.println("---- mcep timeline result:");
+			System.out.println("Number of files scanned: " + baseNameArray.length);
+			System.out.println("Total speech duration: [" + totalTime + "] samples / ["
+					+ ((double) (totalTime) / (double) (globSampleRate)) + "] seconds.");
+			System.out.println("(Speech duration approximated from EST Track float times: [" + totalDuration + "] seconds.)");
+			System.out.println("Number of frames: [" + numDatagrams + "].");
+			System.out.println("Size of the index: [" + mcepTimeline.getIndex().getNumIdx() + "] ("
+					+ (mcepTimeline.getIndex().getNumIdx() * 16) + " bytes, i.e. "
+					+ new DecimalFormat("#.##").format((double) (mcepTimeline.getIndex().getNumIdx()) * 16.0 / 1048576.0)
+					+ " megs).");
+			System.out.println("---- mcep timeline done.");
+
+		} catch (SecurityException e) {
+			System.err.println("Error: you don't have write access to the target database directory.");
+			return false;
+		} catch (Exception e) {
+			e.printStackTrace();
+			System.err.println(e);
+			return false;
+		}
+
+		return (true);
+	}
+
+	/**
+	 * Provide the progress of computation, in percent, or -1 if that feature is not implemented.
+	 * 
+	 * @return -1 if not implemented, or an integer between 0 and 100.
+	 */
+	public int getProgress() {
+		return percent;
+	}
 
 }

marytts-builder/src/main/java/marytts/tools/voiceimport/OctaveVoiceQualityProcessor.java

@@ -35,327 +35,312 @@ import marytts.signalproc.analysis.VoiceQuality;
 import marytts.util.io.StreamGobbler;
 import marytts.util.MaryUtils;
 
-
 public class OctaveVoiceQualityProcessor extends VoiceImportComponent {
-    
-    protected DatabaseLayout db;
-    private String name = "OctaveVoiceQualityProcessor";
-    protected String snackExtension = ".snack";
-    protected String octaveExtension = ".octave";
-    protected String voiceQualityExtension = ".vq";
-    protected String scriptSnackFileName;
-    protected String scriptOctaveFileName;
-    
 
-    int numVqParams = 5;           // number of voice quality parameters extracted from the sound files:
-                                   // OQG, GOG, SKG, RCG, IC
-      
-    private int percent = 0;
-    //private final String FRAMELENGTH  = "0.01";   // Default for snack
-    //private final String WINDOWLENGTH = "0.025";  // Default for f0 snack ( formants uses a bigger window)
-  
-    public final String SAMPLINGRATE = "OctaveVoiceQualityProcessor.samplingRate";
-    public final String MINPITCH     = "OctaveVoiceQualityProcessor.minPitch";
-    public final String MAXPITCH     = "OctaveVoiceQualityProcessor.maxPitch";
-    public final String FRAMELENGTH  = "OctaveVoiceQualityProcessor.frameLength";
-    public final String WINDOWLENGTH = "OctaveVoiceQualityProcessor.windowLength";
-    public final String NUMFORMANTS  = "OctaveVoiceQualityProcessor.numFormants";
-    public final String LPCORDER     = "OctaveVoiceQualityProcessor.lpcOrder";
-    public final String VQDIR        = "OctaveVoiceQualityProcessor.vqDir";
-    public final String OCTAVEPATH   = "OctaveVoiceQualityProcessor.octavePath";
-    
-    protected void setupHelp()
-    {
-        if (props2Help ==null){
-            props2Help = new TreeMap();
-            props2Help.put(SAMPLINGRATE,"Sampling frequency in Hertz. Default: 16000");
-            props2Help.put(MINPITCH,"minimum value for the pitch (in Hz). Default: female 60, male 40");
-            props2Help.put(MAXPITCH,"maximum value for the pitch (in Hz). Default: female 500, male 400");
-            props2Help.put(FRAMELENGTH,"frame length (in seconds) for VQ calculation Default: 0.005 sec.");
-            props2Help.put(WINDOWLENGTH,"window length (in seconds) for VQ calculation Default: 0.025 sec.");
-            props2Help.put(NUMFORMANTS,"Default 4, maximum 7");
-            props2Help.put(LPCORDER,"Default 12, if NUMFORMANTS=4 min LPCORDER=12\n" +
-                                                "if NUMFORMANTS=5 min LPCORDER=14\n" +
-                                                "if NUMFORMANTS=6 min LPCORDER=16\n" +
-                                                "if NUMFORMANTS=7 min LPCORDER=18\n" );            
-            props2Help.put(VQDIR, "directory containing the voice quality files. Will be created if it does not exist");
-            props2Help.put(OCTAVEPATH, "octave executable path");
-        }
-    }
+	protected DatabaseLayout db;
+	private String name = "OctaveVoiceQualityProcessor";
+	protected String snackExtension = ".snack";
+	protected String octaveExtension = ".octave";
+	protected String voiceQualityExtension = ".vq";
+	protected String scriptSnackFileName;
+	protected String scriptOctaveFileName;
 
-    public final String getName(){
-        return name;
-    }
+	int numVqParams = 5; // number of voice quality parameters extracted from the sound files:
+							// OQG, GOG, SKG, RCG, IC
 
-    @Override
-    protected void initialiseComp()
-    {
-        scriptSnackFileName = db.getProp(db.TEMPDIR) + "snack_call.tcl";
-        scriptOctaveFileName = db.getProp(db.TEMPDIR) + "octave_call.m";
-    }
+	private int percent = 0;
+	// private final String FRAMELENGTH = "0.01"; // Default for snack
+	// private final String WINDOWLENGTH = "0.025"; // Default for f0 snack ( formants uses a bigger window)
 
-    public SortedMap getDefaultProps(DatabaseLayout db){
-        this.db = db;
-        if (props == null){
-            props = new TreeMap();  
-            props.put(SAMPLINGRATE,"16000");
-            if (db.getProp(db.GENDER).equals("female")){
-                props.put(MINPITCH,"60");
-                props.put(MAXPITCH,"400");
-            } else {
-                props.put(MINPITCH,"60");
-                props.put(MAXPITCH,"400");
-            }
-            props.put(FRAMELENGTH,"0.005");
-            props.put(WINDOWLENGTH,"0.025");
-            props.put(NUMFORMANTS,"4");
-            props.put(LPCORDER,"12");
-            props.put(VQDIR, db.getProp(db.ROOTDIR) + "vq" + System.getProperty("file.separator"));
-            props.put(OCTAVEPATH, "/usr/bin/octave");
-        }
-        return props;
-    }
+	public final String SAMPLINGRATE = "OctaveVoiceQualityProcessor.samplingRate";
+	public final String MINPITCH = "OctaveVoiceQualityProcessor.minPitch";
+	public final String MAXPITCH = "OctaveVoiceQualityProcessor.maxPitch";
+	public final String FRAMELENGTH = "OctaveVoiceQualityProcessor.frameLength";
+	public final String WINDOWLENGTH = "OctaveVoiceQualityProcessor.windowLength";
+	public final String NUMFORMANTS = "OctaveVoiceQualityProcessor.numFormants";
+	public final String LPCORDER = "OctaveVoiceQualityProcessor.lpcOrder";
+	public final String VQDIR = "OctaveVoiceQualityProcessor.vqDir";
+	public final String OCTAVEPATH = "OctaveVoiceQualityProcessor.octavePath";
 
-    /**
-     * The standard compute() method of the VoiceImportComponent interface.
-     */
-    public boolean compute() throws Exception {
-        
-        /* In order to get the same number of frames when calculating f0 and formants with snack, we should keep constant the following variables:
-         * -maxpitch 400 for F0 calculation
-         * -minpitch 60 for F0 calculation
-         * -windowlength 0.03 for formants calculation
-         * -framelength should be the same for f0, formants and this SnackVoiceQualityProcessor, this value can be change, ex: 0.005, 0.01 etc.
-         */            
-        File scriptSnack = new File(scriptSnackFileName);
-    
-        if (scriptSnack.exists()) scriptSnack.delete();
-        PrintWriter toScriptSnack = new PrintWriter(new FileWriter(scriptSnack));
-        toScriptSnack.println("# extracting pitch anf formants using snack");
-        toScriptSnack.println("package require snack");
-        toScriptSnack.println("snack::sound s");
-        toScriptSnack.println("s read [lindex $argv 0]");
-        toScriptSnack.println("set fd [open [lindex $argv 1] w]");
-        toScriptSnack.println("set f0 [s pitch -method esps -maxpitch [lindex $argv 2] -minpitch [lindex $argv 3] -framelength [lindex $argv 4] ]");
-        toScriptSnack.println("set f0_length [llength $f0]");
-        //toScriptSnack.println("puts \"f0 length = $f0_length\"");
-        toScriptSnack.println("set formants [s formant -numformants [lindex $argv 5] -lpcorder [lindex $argv 6] -framelength [lindex $argv 4] -windowlength 0.03]");
-        toScriptSnack.println("set formants_length [llength $formants]");
-        //toScriptSnack.println("puts \"formants length = $formants_length\"");
-        toScriptSnack.println("set n 0");
-        toScriptSnack.println("foreach line $f0 {");
-        toScriptSnack.println("puts -nonewline $fd \"[lindex $line 0] \"");
-        toScriptSnack.println("puts $fd [lindex $formants $n]");
-        toScriptSnack.println("incr n");
-        toScriptSnack.println("}");
-        toScriptSnack.println("close $fd");
-        toScriptSnack.println("exit"); 
-        toScriptSnack.close();
+	protected void setupHelp() {
+		if (props2Help == null) {
+			props2Help = new TreeMap();
+			props2Help.put(SAMPLINGRATE, "Sampling frequency in Hertz. Default: 16000");
+			props2Help.put(MINPITCH, "minimum value for the pitch (in Hz). Default: female 60, male 40");
+			props2Help.put(MAXPITCH, "maximum value for the pitch (in Hz). Default: female 500, male 400");
+			props2Help.put(FRAMELENGTH, "frame length (in seconds) for VQ calculation Default: 0.005 sec.");
+			props2Help.put(WINDOWLENGTH, "window length (in seconds) for VQ calculation Default: 0.025 sec.");
+			props2Help.put(NUMFORMANTS, "Default 4, maximum 7");
+			props2Help.put(LPCORDER, "Default 12, if NUMFORMANTS=4 min LPCORDER=12\n" + "if NUMFORMANTS=5 min LPCORDER=14\n"
+					+ "if NUMFORMANTS=6 min LPCORDER=16\n" + "if NUMFORMANTS=7 min LPCORDER=18\n");
+			props2Help.put(VQDIR, "directory containing the voice quality files. Will be created if it does not exist");
+			props2Help.put(OCTAVEPATH, "octave executable path");
+		}
+	}
 
-              
-        File scriptOctave = new File(scriptOctaveFileName);
-        if (scriptOctave.exists()) scriptOctave.delete();
-        PrintWriter toScriptOctave = new PrintWriter(new FileWriter(scriptOctave));
-        toScriptOctave.println("arg_list = argv ();");        
-        toScriptOctave.println("cd " + db.getProp(db.TEMPDIR));
-        // calculateVoiceQuality(filename, filesnack, gender, par_name, debug);
-        toScriptOctave.println("calculateVoiceQuality(arg_list{1}, arg_list{2}, arg_list{3}, arg_list{4});");
-        toScriptOctave.close();
+	public final String getName() {
+		return name;
+	}
 
-        
-        String[] baseNameArray = bnl.getListAsArray();
-        // to test String[] baseNameArray = {"curious", "u"};
-        System.out.println( "Computing voice quality for " + baseNameArray.length + " utterances." );
+	@Override
+	protected void initialiseComp() {
+		scriptSnackFileName = db.getProp(db.TEMPDIR) + "snack_call.tcl";
+		scriptOctaveFileName = db.getProp(db.TEMPDIR) + "octave_call.m";
+	}
 
-        /* Ensure the existence of the target pitchmark directory */
-        File dir = new File(getProp(VQDIR));
-        if (!dir.exists()) {
-            System.out.println( "Creating the directory [" + getProp(VQDIR) + "]." );
-            dir.mkdir();
-        }        
-              
-        // Some general parameters that apply to all the sound files
-        int samplingRate = Integer.parseInt(getProp(SAMPLINGRATE));         
-        // frameLength and windowLength in samples
-        int frameLength = Math.round(Float.parseFloat(getProp(FRAMELENGTH)) * samplingRate);
-        int windowLength = Math.round(Float.parseFloat(getProp(WINDOWLENGTH)) * samplingRate);
-        
-            
-        /* execute octave and voice quality parameters extraction */        
-        for ( int i = 0; i < baseNameArray.length; i++ ) {
-            percent = 100*i/baseNameArray.length;
-            
-            /* call snack for calculating f0 and formants */
-            String wavFile   = db.getProp(db.WAVDIR) + baseNameArray[i] + db.getProp(db.WAVEXT);
-            String octaveFile = getProp(VQDIR) + baseNameArray[i] + octaveExtension;
-            String snackFile = getProp(VQDIR) + baseNameArray[i] + snackExtension;
-            String vqFile    = getProp(VQDIR) + baseNameArray[i] + voiceQualityExtension;
-                        
-            System.out.println("Writing (snack) f0+formants+bandWidths to " + snackFile);
-            boolean isWindows = true;
-            String strSnackTmp = scriptSnackFileName + " " + wavFile + " " + snackFile + " " + getProp(MAXPITCH) + " " + getProp(MINPITCH)
-                               + " " + getProp(FRAMELENGTH) + " " + getProp(NUMFORMANTS) + " " + getProp(LPCORDER);
-            if (MaryUtils.isWindows())
-                strSnackTmp = "cmd.exe /c " + db.getExternal(db.TCLPATH) + "/tclsh " + strSnackTmp;
-            else
-                strSnackTmp = db.getExternal(db.TCLPATH) + "/tclsh " + strSnackTmp;           
-            //System.out.println("Executing: " + strSnackTmp);                
-            Process snack = Runtime.getRuntime().exec(strSnackTmp);
-            StreamGobbler errorGobbler1 = new StreamGobbler(snack.getErrorStream(), "err");            
-            //read from output stream
-            StreamGobbler outputGobbler1 = new StreamGobbler(snack.getInputStream(), "out");    
-            //start reading from the streams
-            errorGobbler1.start();
-            outputGobbler1.start();
-            //close everything down
-            snack.waitFor();
-            snack.exitValue();
-            
-            /* call octave for calculating VQ parameters */            
-            //System.out.println("Calculating  OQG GOG SKG RCG IC");
-            //TODO: gender does not appear properly
-            String strOctaveTmp = getProp(OCTAVEPATH) + " --silent " + scriptOctaveFileName + " " + wavFile + " " + snackFile + " " 
-                                + getProp(db.GENDER) + " " + octaveFile;            
-            //System.out.println("Executing: " + strOctaveTmp);                
-            Process octave = Runtime.getRuntime().exec(strOctaveTmp);
-            StreamGobbler errorGobbler2 = new StreamGobbler(octave.getErrorStream(), "err");            
-            //read from output stream
-            StreamGobbler outputGobbler2 = new StreamGobbler(octave.getInputStream(), "out");    
-            //start reading from the streams
-            errorGobbler2.start();
-            outputGobbler2.start();
-            //close everything down
-            octave.waitFor();
-            octave.exitValue();
-           
-            // Read the sound file
-            WavReader soundFile = new WavReader(wavFile);           
-            // Check sampling rate of sound file
-            assert samplingRate==soundFile.getSampleRate();
-            
-            // get a wrapper voice quality class for this file           
-            VoiceQuality vq = new VoiceQuality(numVqParams,samplingRate, frameLength/(float)samplingRate,windowLength/(float)samplingRate);
-            
-            readOctaveData(vq, octaveFile);
-           
-            System.out.println("Writing (octave) vq parameters to " + vqFile);
-            vq.writeVqFile(vqFile);
-            
-        }
-        return true;
-    }
-    
-  
-    private void readOctaveData(VoiceQuality vq, String octaveFile) throws IOException
-    {
-        double[][] octaveData = null;
-        int numLines, numData;
-        BufferedReader reader = new BufferedReader(new FileReader(octaveFile));
-        int i, j;
-        try {            
-            String line;
-            String strVal;
-            StringTokenizer s;
-            double value;
-            
-            // find out the number of lines in the file
-            List<String> lines = new ArrayList<String>();
-            while ((line = reader.readLine())!=null){
-                lines.add(line);
-            }
-            numLines = lines.size();
-            numData = vq.params.dimension;
-            octaveData = new double[numData][numLines];
-            for (i=0; i<numLines; i++){
-         
-                strVal = (String)lines.get(i);
-                s = new StringTokenizer(strVal);
-                
-                for (j=0; j<numData; j++)
-                {
-                  if(s.hasMoreTokens())
-                    octaveData[j][i] = Double.parseDouble(s.nextToken());
-                }            
-            }
-            vq.allocate(numLines, octaveData);
-            
-        } catch (IOException ioe) {
-            ioe.printStackTrace();            
-        } catch (NumberFormatException nfe) {
-            nfe.printStackTrace();
-        }
-    }
+	public SortedMap getDefaultProps(DatabaseLayout db) {
+		this.db = db;
+		if (props == null) {
+			props = new TreeMap();
+			props.put(SAMPLINGRATE, "16000");
+			if (db.getProp(db.GENDER).equals("female")) {
+				props.put(MINPITCH, "60");
+				props.put(MAXPITCH, "400");
+			} else {
+				props.put(MINPITCH, "60");
+				props.put(MAXPITCH, "400");
+			}
+			props.put(FRAMELENGTH, "0.005");
+			props.put(WINDOWLENGTH, "0.025");
+			props.put(NUMFORMANTS, "4");
+			props.put(LPCORDER, "12");
+			props.put(VQDIR, db.getProp(db.ROOTDIR) + "vq" + System.getProperty("file.separator"));
+			props.put(OCTAVEPATH, "/usr/bin/octave");
+		}
+		return props;
+	}
 
-    /**
-     * Provide the progress of computation, in percent, or -1 if
-     * that feature is not implemented.
-     * @return -1 if not implemented, or an integer between 0 and 100.
-     */
-    public int getProgress()
-    {
-        return percent;
-    }
-    
-    
-    // to test/compare vq values of several files
-    public static void main1( String[] args ) throws Exception    
-    {
-      
-        String path = "/project/mary/marcela/HMM-voices/arctic_test/vq-octave/";                 
-        String whisperFile = path + "whisper.vq";
-        String modalFile = path + "modal.vq";
-        String creakFile = path + "creak.vq";
-        String harshFile = path + "harsh.vq";
-                
-        VoiceQuality vq1 = new VoiceQuality(); 
-        System.out.println("Reading: " + whisperFile);
-        vq1.readVqFile(whisperFile); 
-        //vq1.printPar();
-        vq1.printMeanStd();
+	/**
+	 * The standard compute() method of the VoiceImportComponent interface.
+	 */
+	public boolean compute() throws Exception {
 
-        VoiceQuality vq2 = new VoiceQuality();        
-        System.out.println("Reading: " + modalFile);
-        vq2.readVqFile(modalFile);  
-        //vq2.printPar();
-        vq2.printMeanStd();
-        
-        VoiceQuality vq3 = new VoiceQuality();
-        System.out.println("Reading: " + creakFile);
-        vq3.readVqFile(creakFile);  
-        //vq3.printPar();
-        vq3.printMeanStd();
-        
-        VoiceQuality vq4 = new VoiceQuality();
-        System.out.println("Reading: " + harshFile);
-        vq4.readVqFile(harshFile);  
-        //vq4.printPar();
-        vq4.printMeanStd();
-        
-    }    
+		/*
+		 * In order to get the same number of frames when calculating f0 and formants with snack, we should keep constant the
+		 * following variables: -maxpitch 400 for F0 calculation -minpitch 60 for F0 calculation -windowlength 0.03 for formants
+		 * calculation -framelength should be the same for f0, formants and this SnackVoiceQualityProcessor, this value can be
+		 * change, ex: 0.005, 0.01 etc.
+		 */
+		File scriptSnack = new File(scriptSnackFileName);
+
+		if (scriptSnack.exists())
+			scriptSnack.delete();
+		PrintWriter toScriptSnack = new PrintWriter(new FileWriter(scriptSnack));
+		toScriptSnack.println("# extracting pitch anf formants using snack");
+		toScriptSnack.println("package require snack");
+		toScriptSnack.println("snack::sound s");
+		toScriptSnack.println("s read [lindex $argv 0]");
+		toScriptSnack.println("set fd [open [lindex $argv 1] w]");
+		toScriptSnack
+				.println("set f0 [s pitch -method esps -maxpitch [lindex $argv 2] -minpitch [lindex $argv 3] -framelength [lindex $argv 4] ]");
+		toScriptSnack.println("set f0_length [llength $f0]");
+		// toScriptSnack.println("puts \"f0 length = $f0_length\"");
+		toScriptSnack
+				.println("set formants [s formant -numformants [lindex $argv 5] -lpcorder [lindex $argv 6] -framelength [lindex $argv 4] -windowlength 0.03]");
+		toScriptSnack.println("set formants_length [llength $formants]");
+		// toScriptSnack.println("puts \"formants length = $formants_length\"");
+		toScriptSnack.println("set n 0");
+		toScriptSnack.println("foreach line $f0 {");
+		toScriptSnack.println("puts -nonewline $fd \"[lindex $line 0] \"");
+		toScriptSnack.println("puts $fd [lindex $formants $n]");
+		toScriptSnack.println("incr n");
+		toScriptSnack.println("}");
+		toScriptSnack.println("close $fd");
+		toScriptSnack.println("exit");
+		toScriptSnack.close();
+
+		File scriptOctave = new File(scriptOctaveFileName);
+		if (scriptOctave.exists())
+			scriptOctave.delete();
+		PrintWriter toScriptOctave = new PrintWriter(new FileWriter(scriptOctave));
+		toScriptOctave.println("arg_list = argv ();");
+		toScriptOctave.println("cd " + db.getProp(db.TEMPDIR));
+		// calculateVoiceQuality(filename, filesnack, gender, par_name, debug);
+		toScriptOctave.println("calculateVoiceQuality(arg_list{1}, arg_list{2}, arg_list{3}, arg_list{4});");
+		toScriptOctave.close();
+
+		String[] baseNameArray = bnl.getListAsArray();
+		// to test String[] baseNameArray = {"curious", "u"};
+		System.out.println("Computing voice quality for " + baseNameArray.length + " utterances.");
+
+		/* Ensure the existence of the target pitchmark directory */
+		File dir = new File(getProp(VQDIR));
+		if (!dir.exists()) {
+			System.out.println("Creating the directory [" + getProp(VQDIR) + "].");
+			dir.mkdir();
+		}
+
+		// Some general parameters that apply to all the sound files
+		int samplingRate = Integer.parseInt(getProp(SAMPLINGRATE));
+		// frameLength and windowLength in samples
+		int frameLength = Math.round(Float.parseFloat(getProp(FRAMELENGTH)) * samplingRate);
+		int windowLength = Math.round(Float.parseFloat(getProp(WINDOWLENGTH)) * samplingRate);
+
+		/* execute octave and voice quality parameters extraction */
+		for (int i = 0; i < baseNameArray.length; i++) {
+			percent = 100 * i / baseNameArray.length;
+
+			/* call snack for calculating f0 and formants */
+			String wavFile = db.getProp(db.WAVDIR) + baseNameArray[i] + db.getProp(db.WAVEXT);
+			String octaveFile = getProp(VQDIR) + baseNameArray[i] + octaveExtension;
+			String snackFile = getProp(VQDIR) + baseNameArray[i] + snackExtension;
+			String vqFile = getProp(VQDIR) + baseNameArray[i] + voiceQualityExtension;
+
+			System.out.println("Writing (snack) f0+formants+bandWidths to " + snackFile);
+			boolean isWindows = true;
+			String strSnackTmp = scriptSnackFileName + " " + wavFile + " " + snackFile + " " + getProp(MAXPITCH) + " "
+					+ getProp(MINPITCH) + " " + getProp(FRAMELENGTH) + " " + getProp(NUMFORMANTS) + " " + getProp(LPCORDER);
+			if (MaryUtils.isWindows())
+				strSnackTmp = "cmd.exe /c " + db.getExternal(db.TCLPATH) + "/tclsh " + strSnackTmp;
+			else
+				strSnackTmp = db.getExternal(db.TCLPATH) + "/tclsh " + strSnackTmp;
+			// System.out.println("Executing: " + strSnackTmp);
+			Process snack = Runtime.getRuntime().exec(strSnackTmp);
+			StreamGobbler errorGobbler1 = new StreamGobbler(snack.getErrorStream(), "err");
+			// read from output stream
+			StreamGobbler outputGobbler1 = new StreamGobbler(snack.getInputStream(), "out");
+			// start reading from the streams
+			errorGobbler1.start();
+			outputGobbler1.start();
+			// close everything down
+			snack.waitFor();
+			snack.exitValue();
+
+			/* call octave for calculating VQ parameters */
+			// System.out.println("Calculating  OQG GOG SKG RCG IC");
+			// TODO: gender does not appear properly
+			String strOctaveTmp = getProp(OCTAVEPATH) + " --silent " + scriptOctaveFileName + " " + wavFile + " " + snackFile
+					+ " " + getProp(db.GENDER) + " " + octaveFile;
+			// System.out.println("Executing: " + strOctaveTmp);
+			Process octave = Runtime.getRuntime().exec(strOctaveTmp);
+			StreamGobbler errorGobbler2 = new StreamGobbler(octave.getErrorStream(), "err");
+			// read from output stream
+			StreamGobbler outputGobbler2 = new StreamGobbler(octave.getInputStream(), "out");
+			// start reading from the streams
+			errorGobbler2.start();
+			outputGobbler2.start();
+			// close everything down
+			octave.waitFor();
+			octave.exitValue();
+
+			// Read the sound file
+			WavReader soundFile = new WavReader(wavFile);
+			// Check sampling rate of sound file
+			assert samplingRate == soundFile.getSampleRate();
+
+			// get a wrapper voice quality class for this file
+			VoiceQuality vq = new VoiceQuality(numVqParams, samplingRate, frameLength / (float) samplingRate, windowLength
+					/ (float) samplingRate);
+
+			readOctaveData(vq, octaveFile);
+
+			System.out.println("Writing (octave) vq parameters to " + vqFile);
+			vq.writeVqFile(vqFile);
+
+		}
+		return true;
+	}
+
+	private void readOctaveData(VoiceQuality vq, String octaveFile) throws IOException {
+		double[][] octaveData = null;
+		int numLines, numData;
+		BufferedReader reader = new BufferedReader(new FileReader(octaveFile));
+		int i, j;
+		try {
+			String line;
+			String strVal;
+			StringTokenizer s;
+			double value;
+
+			// find out the number of lines in the file
+			List<String> lines = new ArrayList<String>();
+			while ((line = reader.readLine()) != null) {
+				lines.add(line);
+			}
+			numLines = lines.size();
+			numData = vq.params.dimension;
+			octaveData = new double[numData][numLines];
+			for (i = 0; i < numLines; i++) {
+
+				strVal = (String) lines.get(i);
+				s = new StringTokenizer(strVal);
+
+				for (j = 0; j < numData; j++) {
+					if (s.hasMoreTokens())
+						octaveData[j][i] = Double.parseDouble(s.nextToken());
+				}
+			}
+			vq.allocate(numLines, octaveData);
+
+		} catch (IOException ioe) {
+			ioe.printStackTrace();
+		} catch (NumberFormatException nfe) {
+			nfe.printStackTrace();
+		}
+	}
+
+	/**
+	 * Provide the progress of computation, in percent, or -1 if that feature is not implemented.
+	 * 
+	 * @return -1 if not implemented, or an integer between 0 and 100.
+	 */
+	public int getProgress() {
+		return percent;
+	}
+
+	// to test/compare vq values of several files
+	public static void main1(String[] args) throws Exception {
+
+		String path = "/project/mary/marcela/HMM-voices/arctic_test/vq-octave/";
+		String whisperFile = path + "whisper.vq";
+		String modalFile = path + "modal.vq";
+		String creakFile = path + "creak.vq";
+		String harshFile = path + "harsh.vq";
+
+		VoiceQuality vq1 = new VoiceQuality();
+		System.out.println("Reading: " + whisperFile);
+		vq1.readVqFile(whisperFile);
+		// vq1.printPar();
+		vq1.printMeanStd();
+
+		VoiceQuality vq2 = new VoiceQuality();
+		System.out.println("Reading: " + modalFile);
+		vq2.readVqFile(modalFile);
+		// vq2.printPar();
+		vq2.printMeanStd();
+
+		VoiceQuality vq3 = new VoiceQuality();
+		System.out.println("Reading: " + creakFile);
+		vq3.readVqFile(creakFile);
+		// vq3.printPar();
+		vq3.printMeanStd();
+
+		VoiceQuality vq4 = new VoiceQuality();
+		System.out.println("Reading: " + harshFile);
+		vq4.readVqFile(harshFile);
+		// vq4.printPar();
+		vq4.printMeanStd();
+
+	}
+
+	public static void main(String[] args) throws Exception {
+		/*
+		 * OctaveVoiceQualityProcessor vq = new OctaveVoiceQualityProcessor(); DatabaseLayout db = new DatabaseLayout(vq);
+		 * vq.compute();
+		 */
+		// values extracted with Java program
+		// main1(args);
+
+		String file = "/project/mary/marcela/UnitSel-voices/slt-arctic/vq/curious.vq";
+		VoiceQuality vq1 = new VoiceQuality();
+		System.out.println("Reading: " + file);
+		vq1.readVqFile(file);
+		vq1.printPar();
+		vq1.printMeanStd();
+		// MaryUtils.plot(vq1.getGOG(), "Normal");
+		// vq1.applyZscoreNormalization();
+		// MaryUtils.plot(vq1.getGOG(), "after z-score");
+
+	}
 
-   
-    public static void main( String[] args ) throws Exception { 
-      /*OctaveVoiceQualityProcessor vq = new OctaveVoiceQualityProcessor(); 
-      DatabaseLayout db = new DatabaseLayout(vq);
-      vq.compute();
-      */
-      // values extracted with Java program
-      //main1(args);
-            
-      String file   = "/project/mary/marcela/UnitSel-voices/slt-arctic/vq/curious.vq";
-      VoiceQuality vq1 = new VoiceQuality(); 
-      System.out.println("Reading: " + file);
-      vq1.readVqFile(file); 
-      vq1.printPar();
-      vq1.printMeanStd();
-      //MaryUtils.plot(vq1.getGOG(), "Normal");
-      //vq1.applyZscoreNormalization();
-      //MaryUtils.plot(vq1.getGOG(), "after z-score");
-      
-          
-      
-    }
-    
-    
 }
-

marytts-builder/src/main/java/marytts/tools/voiceimport/PauseDurationTrainer.java

@@ -50,378 +50,380 @@ import weka.core.Instances;
  * 
  */
 public class PauseDurationTrainer extends VoiceImportComponent {
-    
-    /**
-     * Tuple holding list of feature vectors and feature definition.
-     */
-    private class VectorsAndDefinition{
-        public VectorsAndDefinition(List<FeatureVector> fv, FeatureDefinition fd) {
-            this.fv = fv;
-            this.fd = fd;
-        }
-        private List<FeatureVector> fv; 
-        private FeatureDefinition fd;
-        public List<FeatureVector> getFv() {
-            return fv;
-        }
-        public void setFv(List<FeatureVector> fv) {
-            this.fv = fv;
-        }
-        public FeatureDefinition getFd() {
-            return fd;
-        }
-        public void setFd(FeatureDefinition fd) {
-            this.fd = fd;
-        }
-    }
-    
-    // maybe specify in config file?
-    public final String[] featureNames = new String[]{"breakindex","ph_cplace","ph_ctype","next_pos","next_wordbegin_ctype","next_wordbegin_cplace","words_from_phrase_end","words_from_phrase_start"/**/};
-    
-    // feature files used for training ("pause features")
-    public final String FVFILES =    "PauseDurationTrainer.featureDir";
-    // label files providing durations
-    public final String LABFILES =   "PauseDurationTrainer.lab";
-    // resulting trained decision tree
-    public final String TRAINEDTREE ="PauseDurationTrainer.tree";
-    
-    protected DatabaseLayout db = null;
-    
-    private String fvExt = ".pfeats";
-    private String labExt = ".lab";
 
-    
-    public SortedMap<String, String> getDefaultProps(DatabaseLayout db){
-        this.db = db;
-        if (props == null){
-            props = new TreeMap<String, String>();
-            
-            // dir with pause feature files 
-            String pauseFv = System.getProperty(FVFILES);
-            if ( pauseFv == null ) {
-                pauseFv = db.getProp(db.ROOTDIR)
-                +"pausefeatures"
-                +System.getProperty("file.separator");
-            }
-            props.put(FVFILES,pauseFv);
-            
-            // dir with lab files containing the pause durations
-            String labs = System.getProperty(LABFILES);
-            if ( labs == null ) {
-                labs = db.getProp(db.ROOTDIR)
-                +"lab"
-                +System.getProperty("file.separator");
-            }
-            props.put(LABFILES,labs);
-            
-            // resulting decision tree
-            String tree = System.getProperty(TRAINEDTREE);
-            if ( tree == null ) {
-                tree = db.getProp(db.ROOTDIR)
-                +"durations.tree";
-            }
-            props.put(TRAINEDTREE,tree);
-            
-        } 
-        return props;
-    }
-    
-    public boolean compute() throws Exception{
-        
-        // object to store all instances
-        Instances data = null;
-        FeatureDefinition fd = null;
-        
-        // pause durations are added at the end
-        // all of them are collected first
-        // then discretized
-        List<Integer> durs = new ArrayList<Integer>();
-        
-        for (int i=0;i<bnl.getLength();i++){
-            
-            VectorsAndDefinition features = this.readFeaturesFor(bnl.getName(i));
-            
-            if (null == features)
-                continue;
-            
-            List<FeatureVector> vectors = features.getFv();
-            fd = features.getFd();
+	/**
+	 * Tuple holding list of feature vectors and feature definition.
+	 */
+	private class VectorsAndDefinition {
+		public VectorsAndDefinition(List<FeatureVector> fv, FeatureDefinition fd) {
+			this.fv = fv;
+			this.fd = fd;
+		}
 
-            if (data == null)
-                data = initData(fd);
-            
-            // reader for label file.
-            BufferedReader lab = new BufferedReader(new FileReader(getProp(LABFILES)+bnl.getName(i)+labExt));
-            
-            List<String> labSyms = new ArrayList<String>();
-            List<Integer> labDurs = new ArrayList<Integer>();           
-            int prevTime = 0;
-            int currTime = 0;
-          
-            String line;  
-            while ((line = lab.readLine()) != null){
-                if ( line.startsWith("#") ) 
-                    continue;
-                
-                String[] lineLmnts = line.split("\\s+");
-                
-               if ( lineLmnts.length != 3 )
-                   throw new IllegalArgumentException("Expected three columns in label file, got " + lineLmnts.length);
-               
-               labSyms.add(lineLmnts[2]);
-               
-               // collect durations
-               currTime = (int)(1000 * Float.parseFloat(lineLmnts[0]));
-               int dur = currTime - prevTime;
-               labDurs.add(dur);
-               prevTime = currTime;
-               
-            }
-            
-            int symbolFeature = fd.getFeatureIndex("phone");
-            int breakindexFeature = fd.getFeatureIndex("breakindex");
-            
-            int currLabelNr = 0;
-            
-            // treatment of first pause(s)...
-            while ( labSyms.get(currLabelNr).equals("_") )
-                currLabelNr++;
-            
-            for (FeatureVector fv : vectors){
-                
-                String fvSym = fv.getFeatureAsString(symbolFeature, fd);
+		private List<FeatureVector> fv;
+		private FeatureDefinition fd;
 
-                // all pauses on feature vector side are ignored, they are captured within boundary treatment
-                if (fvSym.equals("_"))
-                    continue;
-                
-                if (!fvSym.equals(labSyms.get(currLabelNr))) throw new IllegalArgumentException(
-                        "Phone symbol of label file ("+fvSym+") and of feature vector ("+labSyms.get(currLabelNr)+") don't correspond. Run CorrectedTranscriptionAligner first.");
+		public List<FeatureVector> getFv() {
+			return fv;
+		}
 
-                int pauseDur = 0;
-                // durations are taken from pauses on label side 
-                if ( (currLabelNr + 1) < labSyms.size() && 
-                        labSyms.get(currLabelNr + 1).equals("_") ){
-                    currLabelNr++;
-                    pauseDur = labDurs.get(currLabelNr);
-                }
-                
-                int bi = fv.getFeatureAsInt(breakindexFeature);
-                if ( bi > 1 ){
-                    // add new training point with fv
-                    durs.add(pauseDur);
-                    data.add(createInstance(data,fd,fv));
+		public void setFv(List<FeatureVector> fv) {
+			this.fv = fv;
+		}
 
-                } // for each break index > 1
-                
-                currLabelNr++;
-            }// for each featurevector
+		public FeatureDefinition getFd() {
+			return fd;
+		}
 
-        } // for each file
-        
-        // set duration target attribute
-        data = enterDurations(data, durs);
-        
-        // train classifier
-        StringPredictionTree wagonTree = trainTree(data, fd); 
-        
-        FileWriter fw = new FileWriter(getProp(TRAINEDTREE));
-        fw.write(wagonTree.toString());
-        fw.close();
+		public void setFd(FeatureDefinition fd) {
+			this.fd = fd;
+		}
+	}
 
-        return true;
-    }
-    
+	// maybe specify in config file?
+	public final String[] featureNames = new String[] { "breakindex", "ph_cplace", "ph_ctype", "next_pos",
+			"next_wordbegin_ctype", "next_wordbegin_cplace", "words_from_phrase_end", "words_from_phrase_start"/**/};
 
-    private StringPredictionTree trainTree(Instances data, FeatureDefinition fd) throws Exception {
-        
-        System.out.println("training duration tree ("+data.numInstances()+" instances) ...");
-        
-        // build the tree without using the J48 wrapper class
-        // standard parameters are: 
-        //binary split selection with minimum x instances at the leaves, tree is pruned, confidence value, subtree raising, cleanup, don't collapse
-        C45PruneableClassifierTree decisionTree = new C45PruneableClassifierTree(new BinC45ModelSelection(2,data,true),true,0.25f,true,true,false);        
-       
-        decisionTree.buildClassifier(data);
-        
-        System.out.println("...done");
-        
-        return TreeConverter.c45toStringPredictionTree(decisionTree,fd,data);
-    }
+	// feature files used for training ("pause features")
+	public final String FVFILES = "PauseDurationTrainer.featureDir";
+	// label files providing durations
+	public final String LABFILES = "PauseDurationTrainer.lab";
+	// resulting trained decision tree
+	public final String TRAINEDTREE = "PauseDurationTrainer.tree";
 
-    private Instances enterDurations(Instances data, List<Integer> durs) {
+	protected DatabaseLayout db = null;
 
-        //System.out.println("discretizing durations...");
-        
-        // now discretize and set target attributes (= pause durations)
-        // for that, first train discretizer
-        GmmDiscretizer discr = GmmDiscretizer.trainDiscretizer(durs, 6, true);
-        
-        // used to store the collected values
-        ArrayList<String> targetVals = new ArrayList<String>();
-        
-        for (int mappedDur : discr.getPossibleValues() ){
-            targetVals.add(mappedDur + "ms");
-        }
-        
-        //FastVector attributeDeclarations = data.;
-        
-        // attribute declaration finished
-        data.insertAttributeAt(new Attribute("target", targetVals), data.numAttributes());
-        
-        for (int i = 0 ; i < durs.size() ; i++ ){
-            
-            Instance currInst = data.instance(i);
-            int dur = durs.get(i);
-            
-            //System.out.println(" mapping " + dur + " to " + discr.discretize(dur) + " - bi:" + data.instance(i).value(data.attribute("breakindex")));
-            
-            currInst.setValue( data.numAttributes() - 1 , discr.discretize(dur) + "ms" );
-            
-        }
-        
-        // Make the last attribute be the class
-        data.setClassIndex(data.numAttributes() - 1);
-        
-        return data;
-    }
+	private String fvExt = ".pfeats";
+	private String labExt = ".lab";
 
-    private Instance createInstance(Instances data, FeatureDefinition fd, FeatureVector fv) {
-        // relevant features + one target
-        Instance currInst = new DenseInstance( data.numAttributes()  );
-        currInst.setDataset(data);
-        
-        // read only relevant features
-        for (String attName : this.featureNames){
-            int featNr = fd.getFeatureIndex(attName);
-            
-            String value = fv.getFeatureAsString( featNr, fd );            
-            currInst.setValue(data.attribute(attName), value);
-        }
-        
-        return currInst;
-    }
+	public SortedMap<String, String> getDefaultProps(DatabaseLayout db) {
+		this.db = db;
+		if (props == null) {
+			props = new TreeMap<String, String>();
 
-    private Instances initData(FeatureDefinition fd) {
-        // this stores the attributes together with allowed values
-        ArrayList<Attribute> attributeDeclarations = new ArrayList<Attribute>();
-        
-        // first declare all the relevant attributes.
-        // Assume that the feature definition and relevant features of the first 
-        // in the list are the same as the others.
+			// dir with pause feature files
+			String pauseFv = System.getProperty(FVFILES);
+			if (pauseFv == null) {
+				pauseFv = db.getProp(db.ROOTDIR) + "pausefeatures" + System.getProperty("file.separator");
+			}
+			props.put(FVFILES, pauseFv);
 
-        for (int attribute = 0; attribute < fd.getNumberOfFeatures() ; attribute ++){
-            
-            String attName = fd.getFeatureName(attribute);
+			// dir with lab files containing the pause durations
+			String labs = System.getProperty(LABFILES);
+			if (labs == null) {
+				labs = db.getProp(db.ROOTDIR) + "lab" + System.getProperty("file.separator");
+			}
+			props.put(LABFILES, labs);
 
-            // skip phone
-            if (attName.equals("phone")){
-                continue;
-            }
+			// resulting decision tree
+			String tree = System.getProperty(TRAINEDTREE);
+			if (tree == null) {
+				tree = db.getProp(db.ROOTDIR) + "durations.tree";
+			}
+			props.put(TRAINEDTREE, tree);
 
-            // ...collect possible values
-            ArrayList<String> attVals = new ArrayList<String>();
-            for (String value : fd.getPossibleValues(attribute)){
-                attVals.add(value);
-            }
-            
-            attributeDeclarations.add(new Attribute(attName, attVals) );
+		}
+		return props;
+	}
 
-        }
-                       
-        // now, create the dataset adding the datapoints
-        return new Instances("pausedurations", attributeDeclarations, 0);
-    }
+	public boolean compute() throws Exception {
 
-    /**
-     * This reads in the features for the symbols in the input (phonemic/automatic) file from a
-     * feature stream stored in textual format.
-     * 
-     * @param featureTable a LineNumberReader from which the feature table is read.
-     * @throws IOException if the input stream is ill-formed
-     */
-    private VectorsAndDefinition readFeatureTable(LineNumberReader featureTable) throws IOException {
-        
-        List<FeatureVector> featureVectors = new ArrayList<FeatureVector>();
-        
-        // read the beginning of the file, containing the feature definition
-        FeatureDefinition fd = new FeatureDefinition(featureTable, false);
+		// object to store all instances
+		Instances data = null;
+		FeatureDefinition fd = null;
 
-        try{
-            // for later checks, get index of phone identity feature
-            fd.getFeatureIndex("phone");
-            fd.getFeatureIndex("breakindex");
-        } catch ( IllegalArgumentException e ) {
-            throw new IOException("Unexpected FeatureDefinition: Does not contain the features 'phone' and 'breakindex'.");
-        }
-        
-        // skip section with string representation
-        while (!featureTable.readLine().equals("")){}
+		// pause durations are added at the end
+		// all of them are collected first
+		// then discretized
+		List<Integer> durs = new ArrayList<Integer>();
 
-        // now, read the features line by line
-        String line = "";
-        
-        while ((line = featureTable.readLine()) != null) {
+		for (int i = 0; i < bnl.getLength(); i++) {
 
-            FeatureVector fv;
-            try{
-                fv = fd.toFeatureVector(0, line);
-            } catch (Exception e) {
-                e.printStackTrace();
-                throw new IOException("Unexpected Input in line " + String.valueOf( featureTable.getLineNumber() ));
-            }
-            
-            featureVectors.add(fv);
-            
-        }
-        
-        return new VectorsAndDefinition(featureVectors,fd);
-    }
+			VectorsAndDefinition features = this.readFeaturesFor(bnl.getName(i));
 
+			if (null == features)
+				continue;
 
-    /**
-     * This reads in some pause feature file and returns feature vectors
-     * 
-     * 
-     * @param basename
-     * @return
-     * @throws IOException
-     */
-    private VectorsAndDefinition readFeaturesFor(String basename) throws IOException
-    {
-        FileInputStream fis;
-        
-        // First, test if there is a corresponding .rawmaryxml file in textdir:
-        File fvFile = new File(getProp(FVFILES) + basename + fvExt );
-        if (fvFile.exists()) {
-            fis = new FileInputStream(fvFile);
-        } else {
-            return null;
-        }
+			List<FeatureVector> vectors = features.getFv();
+			fd = features.getFd();
 
-        System.out.println("processing " + getProp(FVFILES)+basename+fvExt);
+			if (data == null)
+				data = initData(fd);
 
-        // didn't work ... FeatureFileReader ffr = new FeatureFileReader();
-        LineNumberReader lnr = new LineNumberReader(new InputStreamReader(fis));
-               
-        return readFeatureTable(lnr);
-    }
-    
-    public String getName() {
-        return "PauseDurationTrainer";
-    }
+			// reader for label file.
+			BufferedReader lab = new BufferedReader(new FileReader(getProp(LABFILES) + bnl.getName(i) + labExt));
 
-    @Override
-    public int getProgress() {
-        return 0;
-    }
+			List<String> labSyms = new ArrayList<String>();
+			List<Integer> labDurs = new ArrayList<Integer>();
+			int prevTime = 0;
+			int currTime = 0;
 
-    protected void setupHelp() {
-        props2Help = new TreeMap<String, String>();
-        props2Help.put(FVFILES,"Directory containing the pause feature files.");
-        props2Help.put(LABFILES, "Directory containing label files from which pause durations are taken.");
-        props2Help.put(TRAINEDTREE, "Result of training.");
-     }
+			String line;
+			while ((line = lab.readLine()) != null) {
+				if (line.startsWith("#"))
+					continue;
+
+				String[] lineLmnts = line.split("\\s+");
+
+				if (lineLmnts.length != 3)
+					throw new IllegalArgumentException("Expected three columns in label file, got " + lineLmnts.length);
+
+				labSyms.add(lineLmnts[2]);
+
+				// collect durations
+				currTime = (int) (1000 * Float.parseFloat(lineLmnts[0]));
+				int dur = currTime - prevTime;
+				labDurs.add(dur);
+				prevTime = currTime;
+
+			}
+
+			int symbolFeature = fd.getFeatureIndex("phone");
+			int breakindexFeature = fd.getFeatureIndex("breakindex");
+
+			int currLabelNr = 0;
+
+			// treatment of first pause(s)...
+			while (labSyms.get(currLabelNr).equals("_"))
+				currLabelNr++;
+
+			for (FeatureVector fv : vectors) {
+
+				String fvSym = fv.getFeatureAsString(symbolFeature, fd);
+
+				// all pauses on feature vector side are ignored, they are captured within boundary treatment
+				if (fvSym.equals("_"))
+					continue;
+
+				if (!fvSym.equals(labSyms.get(currLabelNr)))
+					throw new IllegalArgumentException("Phone symbol of label file (" + fvSym + ") and of feature vector ("
+							+ labSyms.get(currLabelNr) + ") don't correspond. Run CorrectedTranscriptionAligner first.");
+
+				int pauseDur = 0;
+				// durations are taken from pauses on label side
+				if ((currLabelNr + 1) < labSyms.size() && labSyms.get(currLabelNr + 1).equals("_")) {
+					currLabelNr++;
+					pauseDur = labDurs.get(currLabelNr);
+				}
+
+				int bi = fv.getFeatureAsInt(breakindexFeature);
+				if (bi > 1) {
+					// add new training point with fv
+					durs.add(pauseDur);
+					data.add(createInstance(data, fd, fv));
+
+				} // for each break index > 1
+
+				currLabelNr++;
+			}// for each featurevector
+
+		} // for each file
+
+		// set duration target attribute
+		data = enterDurations(data, durs);
+
+		// train classifier
+		StringPredictionTree wagonTree = trainTree(data, fd);
+
+		FileWriter fw = new FileWriter(getProp(TRAINEDTREE));
+		fw.write(wagonTree.toString());
+		fw.close();
+
+		return true;
+	}
+
+	private StringPredictionTree trainTree(Instances data, FeatureDefinition fd) throws Exception {
+
+		System.out.println("training duration tree (" + data.numInstances() + " instances) ...");
+
+		// build the tree without using the J48 wrapper class
+		// standard parameters are:
+		// binary split selection with minimum x instances at the leaves, tree is pruned, confidence value, subtree raising,
+		// cleanup, don't collapse
+		C45PruneableClassifierTree decisionTree = new C45PruneableClassifierTree(new BinC45ModelSelection(2, data, true), true,
+				0.25f, true, true, false);
+
+		decisionTree.buildClassifier(data);
+
+		System.out.println("...done");
+
+		return TreeConverter.c45toStringPredictionTree(decisionTree, fd, data);
+	}
+
+	private Instances enterDurations(Instances data, List<Integer> durs) {
+
+		// System.out.println("discretizing durations...");
+
+		// now discretize and set target attributes (= pause durations)
+		// for that, first train discretizer
+		GmmDiscretizer discr = GmmDiscretizer.trainDiscretizer(durs, 6, true);
+
+		// used to store the collected values
+		ArrayList<String> targetVals = new ArrayList<String>();
+
+		for (int mappedDur : discr.getPossibleValues()) {
+			targetVals.add(mappedDur + "ms");
+		}
+
+		// FastVector attributeDeclarations = data.;
+
+		// attribute declaration finished
+		data.insertAttributeAt(new Attribute("target", targetVals), data.numAttributes());
+
+		for (int i = 0; i < durs.size(); i++) {
+
+			Instance currInst = data.instance(i);
+			int dur = durs.get(i);
+
+			// System.out.println(" mapping " + dur + " to " + discr.discretize(dur) + " - bi:" +
+			// data.instance(i).value(data.attribute("breakindex")));
+
+			currInst.setValue(data.numAttributes() - 1, discr.discretize(dur) + "ms");
+
+		}
+
+		// Make the last attribute be the class
+		data.setClassIndex(data.numAttributes() - 1);
+
+		return data;
+	}
+
+	private Instance createInstance(Instances data, FeatureDefinition fd, FeatureVector fv) {
+		// relevant features + one target
+		Instance currInst = new DenseInstance(data.numAttributes());
+		currInst.setDataset(data);
+
+		// read only relevant features
+		for (String attName : this.featureNames) {
+			int featNr = fd.getFeatureIndex(attName);
+
+			String value = fv.getFeatureAsString(featNr, fd);
+			currInst.setValue(data.attribute(attName), value);
+		}
+
+		return currInst;
+	}
+
+	private Instances initData(FeatureDefinition fd) {
+		// this stores the attributes together with allowed values
+		ArrayList<Attribute> attributeDeclarations = new ArrayList<Attribute>();
+
+		// first declare all the relevant attributes.
+		// Assume that the feature definition and relevant features of the first
+		// in the list are the same as the others.
+
+		for (int attribute = 0; attribute < fd.getNumberOfFeatures(); attribute++) {
+
+			String attName = fd.getFeatureName(attribute);
+
+			// skip phone
+			if (attName.equals("phone")) {
+				continue;
+			}
+
+			// ...collect possible values
+			ArrayList<String> attVals = new ArrayList<String>();
+			for (String value : fd.getPossibleValues(attribute)) {
+				attVals.add(value);
+			}
+
+			attributeDeclarations.add(new Attribute(attName, attVals));
+
+		}
+
+		// now, create the dataset adding the datapoints
+		return new Instances("pausedurations", attributeDeclarations, 0);
+	}
+
+	/**
+	 * This reads in the features for the symbols in the input (phonemic/automatic) file from a feature stream stored in textual
+	 * format.
+	 * 
+	 * @param featureTable
+	 *            a LineNumberReader from which the feature table is read.
+	 * @throws IOException
+	 *             if the input stream is ill-formed
+	 */
+	private VectorsAndDefinition readFeatureTable(LineNumberReader featureTable) throws IOException {
+
+		List<FeatureVector> featureVectors = new ArrayList<FeatureVector>();
+
+		// read the beginning of the file, containing the feature definition
+		FeatureDefinition fd = new FeatureDefinition(featureTable, false);
+
+		try {
+			// for later checks, get index of phone identity feature
+			fd.getFeatureIndex("phone");
+			fd.getFeatureIndex("breakindex");
+		} catch (IllegalArgumentException e) {
+			throw new IOException("Unexpected FeatureDefinition: Does not contain the features 'phone' and 'breakindex'.");
+		}
+
+		// skip section with string representation
+		while (!featureTable.readLine().equals("")) {
+		}
+
+		// now, read the features line by line
+		String line = "";
+
+		while ((line = featureTable.readLine()) != null) {
+
+			FeatureVector fv;
+			try {
+				fv = fd.toFeatureVector(0, line);
+			} catch (Exception e) {
+				e.printStackTrace();
+				throw new IOException("Unexpected Input in line " + String.valueOf(featureTable.getLineNumber()));
+			}
+
+			featureVectors.add(fv);
+
+		}
+
+		return new VectorsAndDefinition(featureVectors, fd);
+	}
+
+	/**
+	 * This reads in some pause feature file and returns feature vectors
+	 * 
+	 * 
+	 * @param basename
+	 * @return
+	 * @throws IOException
+	 */
+	private VectorsAndDefinition readFeaturesFor(String basename) throws IOException {
+		FileInputStream fis;
+
+		// First, test if there is a corresponding .rawmaryxml file in textdir:
+		File fvFile = new File(getProp(FVFILES) + basename + fvExt);
+		if (fvFile.exists()) {
+			fis = new FileInputStream(fvFile);
+		} else {
+			return null;
+		}
+
+		System.out.println("processing " + getProp(FVFILES) + basename + fvExt);
+
+		// didn't work ... FeatureFileReader ffr = new FeatureFileReader();
+		LineNumberReader lnr = new LineNumberReader(new InputStreamReader(fis));
+
+		return readFeatureTable(lnr);
+	}
+
+	public String getName() {
+		return "PauseDurationTrainer";
+	}
+
+	@Override
+	public int getProgress() {
+		return 0;
+	}
+
+	protected void setupHelp() {
+		props2Help = new TreeMap<String, String>();
+		props2Help.put(FVFILES, "Directory containing the pause feature files.");
+		props2Help.put(LABFILES, "Directory containing label files from which pause durations are taken.");
+		props2Help.put(TRAINEDTREE, "Result of training.");
+	}
 
-    
 }

marytts-builder/src/main/java/marytts/tools/voiceimport/PhoneFeatureFileWriter.java

@@ -42,210 +42,198 @@ import marytts.unitselection.data.FeatureFileReader;
 import marytts.unitselection.data.UnitFileReader;
 import marytts.util.data.MaryHeader;
 
+public class PhoneFeatureFileWriter extends VoiceImportComponent {
+	protected File maryDir;
+	protected FeatureDefinition featureDefinition;
+	protected int percent = 0;
+	protected String featureExt = ".pfeats";
+	protected UnitFileReader unitFileReader;
 
-public class PhoneFeatureFileWriter extends VoiceImportComponent
-{
-    protected File maryDir;
-    protected FeatureDefinition featureDefinition;
-    protected int percent = 0;
-    protected String featureExt = ".pfeats";
-    protected UnitFileReader unitFileReader;
-    
-    protected String name = "PhoneFeatureFileWriter";
-    
-    public String FEATUREDIR = "PhoneFeatureFileWriter.featureDir";
-    public String FEATUREFILE = "PhoneFeatureFileWriter.featureFile";
-    public String UNITFILE = "PhoneFeatureFileWriter.unitFile";
-    public String WEIGHTSFILE = "PhoneFeatureFileWriter.weightsFile";
-    
-    public String getName()
-    {
-        return name;
-    }
-    
-    @Override
-    protected void initialiseComp()
-    {        
-        File unitfeatureDir = new File(getProp(FEATUREDIR));
-        if (!unitfeatureDir.exists()){
-            System.out.print(FEATUREDIR+" "+getProp(FEATUREDIR)
-                    +" does not exist; ");
-            if (!unitfeatureDir.mkdir()){
-                throw new Error("Could not create FEATUREDIR");
-            }
-            System.out.print("Created successfully.\n");
-        }  
-    }
-    
-     public SortedMap getDefaultProps(DatabaseLayout db){
-       if (props == null){
-           props = new TreeMap();
-           props.put(FEATUREDIR, db.getProp(db.ROOTDIR)
-                        +"phonefeatures"
-                        +System.getProperty("file.separator"));
-           props.put(FEATUREFILE, db.getProp(db.FILEDIR)
-                        +"phoneFeatures"+db.getProp(db.MARYEXT));
-           props.put(UNITFILE, db.getProp(db.FILEDIR)
-                        +"phoneUnits"+db.getProp(db.MARYEXT));
-           props.put(WEIGHTSFILE, db.getProp(db.CONFIGDIR)
-                        +"phoneUnitFeatureDefinition.txt");
-       }
-       return props;
-     }
-     
-     protected void setupHelp(){
-         props2Help = new TreeMap();
-         props2Help.put(FEATUREDIR, "directory containing the phone features");
-         props2Help.put(FEATUREFILE, "file containing all phone units and their target cost features."
-                 +"Will be created by this module");
-         props2Help.put(UNITFILE, "file containing all phone units");
-         props2Help.put(WEIGHTSFILE, "file containing the list of phone target cost features, their values and weights");
-         
-     }
-    
-    @Override
-    public boolean compute() throws IOException, MaryConfigurationException
-    {
-        //make sure that we have a featureweightsfile
-        File featWeights = new File(getProp(WEIGHTSFILE));
-        if (!featWeights.exists() || featWeights.length() == 0){
-            PrintWriter featWeightsOut = new PrintWriter(new OutputStreamWriter(new FileOutputStream(featWeights), "UTF-8"), true);
-            BufferedReader uttFeats = new BufferedReader(new InputStreamReader(new FileInputStream(new File(getProp(FEATUREDIR)
-                    + bnl.getName(0) + featureExt)), "UTF-8"));
-            FeatureDefinition featDef = new FeatureDefinition(uttFeats, false); // false: do not read weights
-            uttFeats.close();
-            featDef.generateFeatureWeightsFile(featWeightsOut);
-        }
-        System.out.println("Featurefile writer started.");
-        unitFileReader = new UnitFileReader( getProp(UNITFILE) );
-        featureDefinition = new FeatureDefinition(new BufferedReader(new InputStreamReader(new FileInputStream(getProp(WEIGHTSFILE)), "UTF-8")), true); // true: read weights
+	protected String name = "PhoneFeatureFileWriter";
 
-        DataOutputStream out = new DataOutputStream(new BufferedOutputStream(new FileOutputStream(getProp(FEATUREFILE))));
-        writeHeaderTo(out);
-        writeUnitFeaturesTo(out);
-        out.close();
-        System.out.println("Number of processed units: " + unitFileReader.getNumberOfUnits() );
+	public String FEATUREDIR = "PhoneFeatureFileWriter.featureDir";
+	public String FEATUREFILE = "PhoneFeatureFileWriter.featureFile";
+	public String UNITFILE = "PhoneFeatureFileWriter.unitFile";
+	public String WEIGHTSFILE = "PhoneFeatureFileWriter.weightsFile";
 
-        FeatureFileReader tester = FeatureFileReader.getFeatureFileReader(getProp(FEATUREFILE));
-        int unitsOnDisk = tester.getNumberOfUnits();
-        if (unitsOnDisk == unitFileReader.getNumberOfUnits()) {
-            System.out.println("Can read right number of units");
-            return true;
-        } else {
-            System.out.println("Read wrong number of units: "+unitsOnDisk);
-            return false;
-        }
-    }
-    
-    /**
-     * @param out
-     * @throws IOException
-     * @throws UnsupportedEncodingException
-     * @throws FileNotFoundException
-     */
-    protected void writeUnitFeaturesTo(DataOutput out) throws IOException, UnsupportedEncodingException, FileNotFoundException {
-        int numUnits = unitFileReader.getNumberOfUnits();
-        out.writeInt( numUnits );
-        System.out.println("Number of units : "+numUnits);
-        int index = 0; // the unique index number of units in the unit file
-        // Dummy feature vector corresponding to an edge unit:
-        FeatureVector start = featureDefinition.createEdgeFeatureVector(0, true);
-        FeatureVector end = featureDefinition.createEdgeFeatureVector(0, false);
-        // Loop over all utterances
-        for (int i=0; i<bnl.getLength(); i++) {
-            percent = 100*i/bnl.getLength();
-            System.out.print( "    " + bnl.getName(i) + " : Entering at index (" + index + ") -- " );
-            BufferedReader uttFeats = new BufferedReader(new InputStreamReader(new FileInputStream(new File( getProp(FEATUREDIR)+ bnl.getName(i) + featureExt )), "UTF-8"));
-            FeatureDefinition uttFeatDefinition = new FeatureDefinition(uttFeats, false); // false: do not read weights
-            if (!uttFeatDefinition.featureEquals(featureDefinition)) {
-                throw new IllegalArgumentException("Features in file "+bnl.getName(i)+" do not match definition file "+getProp(WEIGHTSFILE)
-                        + " because:\n"
-                        + uttFeatDefinition.featureEqualsAnalyse(featureDefinition) );
-            }
-            // skip the clear text section: read until an empty line occurs
-            String line;
-            while ((line = uttFeats.readLine()) != null) {
-                if (line.trim().equals("")) break;
-            }
-            // Check the index consistency
-            if ( index > numUnits ) {
-                throw new IOException("Inconsistency between feature files and unit file: " +
-                        "the reached index [" + index+"] is bigger than the number of units in the unit file [" + numUnits + "] !");
-            }
-            // Empty entry corresponding to start of utterance:
-            if (!unitFileReader.isEdgeUnit(index)) {
-                // System.out.println( "Unit [" + index + "] : StarTime [" + unitFileReader.getStartTime(i) + "] Duration [" + unitFileReader.getDuration(i) + "]." );
-                throw new IOException("Inconsistency between feature files and unit file: Unit "
-                        +index+" should correspond to start of file "+bnl.getName(i)
-                        +", but is not an edge unit!");
-            }
-            start.writeTo(out);
-            index++;
-            // Check the index consistency
-            if ( index > numUnits ) {
-                throw new IOException("Inconsistency between feature files and unit file: " +
-                        "the reached index [" + index+"] is bigger than the number of units in the unit file [" + numUnits + "] !");
-            }
-            // read the binary section, and write it
-            while ((line = uttFeats.readLine()) != null) {
-                if (line.trim().equals("")) break;
-                FeatureVector fv = featureDefinition.toFeatureVector(0, line);
-                if (unitFileReader.isEdgeUnit(index)) {
-                    throw new IOException("Inconsistency between feature files and unit file: Unit "
-                            +index+"("+fv.getFeatureAsString(0,featureDefinition)+") should correspond to feature line '"+line+"' of file "+bnl.getName(i)
-                            +", but is an edge unit!");
-                }
-                fv.writeTo(out);
-                index++;
-            }
-            // Check the index consistency
-            if ( index > numUnits ) {
-                throw new IOException("Inconsistency between feature files and unit file: " +
-                        "the reached index [" + index+"] is bigger than the number of units in the unit file [" + numUnits + "] !");
-            }
-            if (!unitFileReader.isEdgeUnit(index)) {
-                throw new IOException("Inconsistency between feature files and unit file: Unit "
-                        +index+" should correspond to end of file "+bnl.getName(i)
-                        +", but is not an edge unit!");
-            }
-            end.writeTo(out);
-            index++;
-            System.out.println( "Exiting at index (" + index + ")." );
-            System.out.flush();
-            uttFeats.close();
-        }
-    }
+	public String getName() {
+		return name;
+	}
 
-    /**
-     * Write the header of this feature file to the given DataOutput
-     * @param out
-     * @throws IOException
-     */
-    protected void writeHeaderTo(DataOutput out) throws IOException {
-        new MaryHeader(MaryHeader.UNITFEATS).writeTo(out);
-        featureDefinition.writeBinaryTo(out);
-    }
+	@Override
+	protected void initialiseComp() {
+		File unitfeatureDir = new File(getProp(FEATUREDIR));
+		if (!unitfeatureDir.exists()) {
+			System.out.print(FEATUREDIR + " " + getProp(FEATUREDIR) + " does not exist; ");
+			if (!unitfeatureDir.mkdir()) {
+				throw new Error("Could not create FEATUREDIR");
+			}
+			System.out.print("Created successfully.\n");
+		}
+	}
 
-    /**
-     * Provide the progress of computation, in percent, or -1 if
-     * that feature is not implemented.
-     * @return -1 if not implemented, or an integer between 0 and 100.
-     */
-    public int getProgress()
-    {
-        return percent;
-    }
+	public SortedMap getDefaultProps(DatabaseLayout db) {
+		if (props == null) {
+			props = new TreeMap();
+			props.put(FEATUREDIR, db.getProp(db.ROOTDIR) + "phonefeatures" + System.getProperty("file.separator"));
+			props.put(FEATUREFILE, db.getProp(db.FILEDIR) + "phoneFeatures" + db.getProp(db.MARYEXT));
+			props.put(UNITFILE, db.getProp(db.FILEDIR) + "phoneUnits" + db.getProp(db.MARYEXT));
+			props.put(WEIGHTSFILE, db.getProp(db.CONFIGDIR) + "phoneUnitFeatureDefinition.txt");
+		}
+		return props;
+	}
 
-    /**
-     * @param args
-     */
-    public static void main(String[] args) throws Exception
-    {
-        PhoneFeatureFileWriter ffw = new PhoneFeatureFileWriter();
-        DatabaseLayout db = new DatabaseLayout(ffw);
-        ffw.compute();
-    }
+	protected void setupHelp() {
+		props2Help = new TreeMap();
+		props2Help.put(FEATUREDIR, "directory containing the phone features");
+		props2Help.put(FEATUREFILE, "file containing all phone units and their target cost features."
+				+ "Will be created by this module");
+		props2Help.put(UNITFILE, "file containing all phone units");
+		props2Help.put(WEIGHTSFILE, "file containing the list of phone target cost features, their values and weights");
 
+	}
+
+	@Override
+	public boolean compute() throws IOException, MaryConfigurationException {
+		// make sure that we have a featureweightsfile
+		File featWeights = new File(getProp(WEIGHTSFILE));
+		if (!featWeights.exists() || featWeights.length() == 0) {
+			PrintWriter featWeightsOut = new PrintWriter(new OutputStreamWriter(new FileOutputStream(featWeights), "UTF-8"), true);
+			BufferedReader uttFeats = new BufferedReader(new InputStreamReader(new FileInputStream(new File(getProp(FEATUREDIR)
+					+ bnl.getName(0) + featureExt)), "UTF-8"));
+			FeatureDefinition featDef = new FeatureDefinition(uttFeats, false); // false: do not read weights
+			uttFeats.close();
+			featDef.generateFeatureWeightsFile(featWeightsOut);
+		}
+		System.out.println("Featurefile writer started.");
+		unitFileReader = new UnitFileReader(getProp(UNITFILE));
+		featureDefinition = new FeatureDefinition(new BufferedReader(new InputStreamReader(new FileInputStream(
+				getProp(WEIGHTSFILE)), "UTF-8")), true); // true: read weights
+
+		DataOutputStream out = new DataOutputStream(new BufferedOutputStream(new FileOutputStream(getProp(FEATUREFILE))));
+		writeHeaderTo(out);
+		writeUnitFeaturesTo(out);
+		out.close();
+		System.out.println("Number of processed units: " + unitFileReader.getNumberOfUnits());
+
+		FeatureFileReader tester = FeatureFileReader.getFeatureFileReader(getProp(FEATUREFILE));
+		int unitsOnDisk = tester.getNumberOfUnits();
+		if (unitsOnDisk == unitFileReader.getNumberOfUnits()) {
+			System.out.println("Can read right number of units");
+			return true;
+		} else {
+			System.out.println("Read wrong number of units: " + unitsOnDisk);
+			return false;
+		}
+	}
+
+	/**
+	 * @param out
+	 * @throws IOException
+	 * @throws UnsupportedEncodingException
+	 * @throws FileNotFoundException
+	 */
+	protected void writeUnitFeaturesTo(DataOutput out) throws IOException, UnsupportedEncodingException, FileNotFoundException {
+		int numUnits = unitFileReader.getNumberOfUnits();
+		out.writeInt(numUnits);
+		System.out.println("Number of units : " + numUnits);
+		int index = 0; // the unique index number of units in the unit file
+		// Dummy feature vector corresponding to an edge unit:
+		FeatureVector start = featureDefinition.createEdgeFeatureVector(0, true);
+		FeatureVector end = featureDefinition.createEdgeFeatureVector(0, false);
+		// Loop over all utterances
+		for (int i = 0; i < bnl.getLength(); i++) {
+			percent = 100 * i / bnl.getLength();
+			System.out.print("    " + bnl.getName(i) + " : Entering at index (" + index + ") -- ");
+			BufferedReader uttFeats = new BufferedReader(new InputStreamReader(new FileInputStream(new File(getProp(FEATUREDIR)
+					+ bnl.getName(i) + featureExt)), "UTF-8"));
+			FeatureDefinition uttFeatDefinition = new FeatureDefinition(uttFeats, false); // false: do not read weights
+			if (!uttFeatDefinition.featureEquals(featureDefinition)) {
+				throw new IllegalArgumentException("Features in file " + bnl.getName(i) + " do not match definition file "
+						+ getProp(WEIGHTSFILE) + " because:\n" + uttFeatDefinition.featureEqualsAnalyse(featureDefinition));
+			}
+			// skip the clear text section: read until an empty line occurs
+			String line;
+			while ((line = uttFeats.readLine()) != null) {
+				if (line.trim().equals(""))
+					break;
+			}
+			// Check the index consistency
+			if (index > numUnits) {
+				throw new IOException("Inconsistency between feature files and unit file: " + "the reached index [" + index
+						+ "] is bigger than the number of units in the unit file [" + numUnits + "] !");
+			}
+			// Empty entry corresponding to start of utterance:
+			if (!unitFileReader.isEdgeUnit(index)) {
+				// System.out.println( "Unit [" + index + "] : StarTime [" + unitFileReader.getStartTime(i) + "] Duration [" +
+				// unitFileReader.getDuration(i) + "]." );
+				throw new IOException("Inconsistency between feature files and unit file: Unit " + index
+						+ " should correspond to start of file " + bnl.getName(i) + ", but is not an edge unit!");
+			}
+			start.writeTo(out);
+			index++;
+			// Check the index consistency
+			if (index > numUnits) {
+				throw new IOException("Inconsistency between feature files and unit file: " + "the reached index [" + index
+						+ "] is bigger than the number of units in the unit file [" + numUnits + "] !");
+			}
+			// read the binary section, and write it
+			while ((line = uttFeats.readLine()) != null) {
+				if (line.trim().equals(""))
+					break;
+				FeatureVector fv = featureDefinition.toFeatureVector(0, line);
+				if (unitFileReader.isEdgeUnit(index)) {
+					throw new IOException("Inconsistency between feature files and unit file: Unit " + index + "("
+							+ fv.getFeatureAsString(0, featureDefinition) + ") should correspond to feature line '" + line
+							+ "' of file " + bnl.getName(i) + ", but is an edge unit!");
+				}
+				fv.writeTo(out);
+				index++;
+			}
+			// Check the index consistency
+			if (index > numUnits) {
+				throw new IOException("Inconsistency between feature files and unit file: " + "the reached index [" + index
+						+ "] is bigger than the number of units in the unit file [" + numUnits + "] !");
+			}
+			if (!unitFileReader.isEdgeUnit(index)) {
+				throw new IOException("Inconsistency between feature files and unit file: Unit " + index
+						+ " should correspond to end of file " + bnl.getName(i) + ", but is not an edge unit!");
+			}
+			end.writeTo(out);
+			index++;
+			System.out.println("Exiting at index (" + index + ").");
+			System.out.flush();
+			uttFeats.close();
+		}
+	}
+
+	/**
+	 * Write the header of this feature file to the given DataOutput
+	 * 
+	 * @param out
+	 * @throws IOException
+	 */
+	protected void writeHeaderTo(DataOutput out) throws IOException {
+		new MaryHeader(MaryHeader.UNITFEATS).writeTo(out);
+		featureDefinition.writeBinaryTo(out);
+	}
+
+	/**
+	 * Provide the progress of computation, in percent, or -1 if that feature is not implemented.
+	 * 
+	 * @return -1 if not implemented, or an integer between 0 and 100.
+	 */
+	public int getProgress() {
+		return percent;
+	}
+
+	/**
+	 * @param args
+	 */
+	public static void main(String[] args) throws Exception {
+		PhoneFeatureFileWriter ffw = new PhoneFeatureFileWriter();
+		DatabaseLayout db = new DatabaseLayout(ffw);
+		ffw.compute();
+	}
 
 }
-

marytts-builder/src/main/java/marytts/tools/voiceimport/PhoneUnitFeatureComputer.java

@@ -31,189 +31,164 @@ import marytts.client.http.MaryHttpClient;
 import marytts.util.http.Address;
 import marytts.util.io.FileUtils;
 
-
-
 /**
- * For the given texts, compute unit features and align them
- * with the given unit labels.
+ * For the given texts, compute unit features and align them with the given unit labels.
+ * 
  * @author schroed
  *
  */
-public class PhoneUnitFeatureComputer extends VoiceImportComponent
-{
-    public static final String PHONEFEATURE = "phone";
-    
-    protected File textDir;
-    protected File unitfeatureDir;
-    protected String featureList;
-    protected String featsExt = ".pfeats";
-    protected String xmlExt = ".xml";
-    protected String locale;
-    protected MaryHttpClient mary;
-    protected String maryInputType;
-    protected String maryOutputType;
-    
-    protected DatabaseLayout db = null;
-    protected int percent = 0;
-    
-    public String FEATUREDIR = "PhoneUnitFeatureComputer.featureDir";
-    public String ALLOPHONES = "PhoneUnitFeatureComputer.allophonesDir";
-    public String FEATURELIST = "PhoneUnitFeatureComputer.featureFile";
-    public String MARYSERVERHOST = "PhoneUnitFeatureComputer.maryServerHost";
-    public String MARYSERVERPORT = "PhoneUnitFeatureComputer.maryServerPort";
-       
-   
-    
-    public String getName(){
-        return "PhoneUnitFeatureComputer";
-    }
-    
-    public static String getMaryXMLHeaderWithInitialBoundary(String locale)
-    {
-        return "<?xml version=\"1.0\" encoding=\"UTF-8\" ?>\n" +
-            "<maryxml version=\"0.4\"\n" +
-            "xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\"\n" +
-            "xmlns=\"http://mary.dfki.de/2002/MaryXML\"\n" +
-            "xml:lang=\"" + locale + "\">\n" +
-            "<boundary  breakindex=\"2\" duration=\"100\"/>\n";
-        
-    }
-    
-    @Override
-    protected void initialiseComp()
-    throws Exception
-    {      
-        locale = db.getProp(db.LOCALE);   
-        
-        mary = null; // initialised only if needed   
-        unitfeatureDir = new File(getProp(FEATUREDIR));
-        if (!unitfeatureDir.exists()){
-            System.out.print(FEATUREDIR+" "+getProp(FEATUREDIR)
-                    +" does not exist; ");
-            if (!unitfeatureDir.mkdir()){
-                throw new Error("Could not create FEATUREDIR");
-            }
-            System.out.print("Created successfully.\n");
-        }
+public class PhoneUnitFeatureComputer extends VoiceImportComponent {
+	public static final String PHONEFEATURE = "phone";
 
-        maryInputType = "ALLOPHONES";
-        maryOutputType = "TARGETFEATURES";
-    }
-     
-     public SortedMap<String,String> getDefaultProps(DatabaseLayout theDb){
-         this.db = theDb;
-         if (props == null){
-             props = new TreeMap<String, String>();
-             props.put(FEATUREDIR, db.getProp(db.ROOTDIR)
-                     +"phonefeatures"
-                     +System.getProperty("file.separator"));
-             props.put(ALLOPHONES, db.getProp(db.ROOTDIR)
-                     +"allophones"
-                     +System.getProperty("file.separator"));
-             props.put(FEATURELIST,
-                     db.getProp(db.CONFIGDIR) + "features.txt");
-             props.put(MARYSERVERHOST,"localhost");
-             props.put(MARYSERVERPORT,"59125");
-         } 
-         
-         return props;
-     }
-     
-     protected void setupHelp(){
-         props2Help = new TreeMap<String, String>();
-         props2Help.put(FEATUREDIR, "directory containing the phone features." 
-                 +"Will be created if it does not exist");
-         props2Help.put(ALLOPHONES, "Directory of corrected allophones files.");
-         props2Help.put(MARYSERVERHOST,"the host were the Mary server is running, default: \"localhost\"");
-         props2Help.put(MARYSERVERPORT,"the port were the Mary server is listening, default: \"59125\"");
-     }
-     
-     public MaryHttpClient getMaryClient() throws IOException
-     {
-        if (mary == null) {
-            try{
-                mary = new MaryHttpClient(new Address(getProp(MARYSERVERHOST), Integer.parseInt(getProp(MARYSERVERPORT))));        
-            } catch (IOException e){
-                throw new IOException("Could not connect to Maryserver at "
-                        +getProp(MARYSERVERHOST)+" "+getProp(MARYSERVERPORT));
-            }
-        }
-        return mary;
-    }
+	protected File textDir;
+	protected File unitfeatureDir;
+	protected String featureList;
+	protected String featsExt = ".pfeats";
+	protected String xmlExt = ".xml";
+	protected String locale;
+	protected MaryHttpClient mary;
+	protected String maryInputType;
+	protected String maryOutputType;
 
-     protected void loadFeatureList()
-     throws IOException
-     {
-         File featureFile = new File(getProp(FEATURELIST));
-         if (!featureFile.exists()) {
-             System.out.println("No feature file: '"+getProp(FEATURELIST)+"'");
-         } else {
-             System.out.println("Loading features from file "+getProp(FEATURELIST));
-             try {
-                 featureList = FileUtils.getFileAsString(featureFile, "UTF-8");
-                 featureList = featureList.replaceAll("\\s+", " ");
-                 // Exclude specific halfphone features if present:
-                 for (String f : HalfPhoneUnitFeatureComputer.HALFPHONE_FEATURES) {
-                     if (featureList.contains(f)) {
-                         featureList = featureList.replaceAll(f, "");
-                     }
-                 }
-                 if (!featureList.contains(PHONEFEATURE)) {
-                     throw new RuntimeException("Feature list does not contain feature '"+PHONEFEATURE+"'. It makes no sense to continue.");
-                 }
-                 if (!featureList.startsWith(PHONEFEATURE)) {
-                     // PHONEFEATURE must be the first one in the list
-                     featureList = featureList.replaceFirst("\\s+"+PHONEFEATURE+"\\s*", " ");
-                     featureList = PHONEFEATURE + " " + featureList;
-                 }
-             } catch (IOException e) {
-                 IOException ioe = new IOException("Cannot read list of features");
-                 ioe.initCause(e);
-                 throw ioe;
-             }
-         }
+	protected DatabaseLayout db = null;
+	protected int percent = 0;
 
-     }
-     
-    public boolean compute() throws IOException
-    {
-        
-        loadFeatureList();
-        
-        textDir = new File(db.getProp(db.TEXTDIR));
-        System.out.println( "Computing unit features for " + bnl.getLength() + " files" );
-        for (int i=0; i<bnl.getLength(); i++) {
-            percent = 100*i/bnl.getLength();
-            computeFeaturesFor( bnl.getName(i) );
-            System.out.println( "    " + bnl.getName(i) );
-        }
-        System.out.println("Finished computing the unit features.");
-        return true;
-    }
+	public String FEATUREDIR = "PhoneUnitFeatureComputer.featureDir";
+	public String ALLOPHONES = "PhoneUnitFeatureComputer.allophonesDir";
+	public String FEATURELIST = "PhoneUnitFeatureComputer.featureFile";
+	public String MARYSERVERHOST = "PhoneUnitFeatureComputer.maryServerHost";
+	public String MARYSERVERPORT = "PhoneUnitFeatureComputer.maryServerPort";
 
-    public void computeFeaturesFor(String basename) throws IOException
-    {
-        File allophoneFile = new File(getProp(ALLOPHONES)
-                + basename + xmlExt);
-        String text = FileUtils.getFileAsString(allophoneFile, "UTF-8");
-        
-        OutputStream os = new BufferedOutputStream(new FileOutputStream(new File( unitfeatureDir, basename + featsExt)));
-        MaryHttpClient maryClient = getMaryClient();
-        
-        maryClient.process(text, maryInputType, maryOutputType, locale, null, null, "", null, featureList, os);
-        os.flush();
-        os.close();
-    }
+	public String getName() {
+		return "PhoneUnitFeatureComputer";
+	}
 
-    /**
-     * Provide the progress of computation, in percent, or -1 if
-     * that feature is not implemented.
-     * @return -1 if not implemented, or an integer between 0 and 100.
-     */
-    public int getProgress()
-    {
-        return percent;
-    }
+	public static String getMaryXMLHeaderWithInitialBoundary(String locale) {
+		return "<?xml version=\"1.0\" encoding=\"UTF-8\" ?>\n" + "<maryxml version=\"0.4\"\n"
+				+ "xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\"\n" + "xmlns=\"http://mary.dfki.de/2002/MaryXML\"\n"
+				+ "xml:lang=\"" + locale + "\">\n" + "<boundary  breakindex=\"2\" duration=\"100\"/>\n";
+
+	}
+
+	@Override
+	protected void initialiseComp() throws Exception {
+		locale = db.getProp(db.LOCALE);
+
+		mary = null; // initialised only if needed
+		unitfeatureDir = new File(getProp(FEATUREDIR));
+		if (!unitfeatureDir.exists()) {
+			System.out.print(FEATUREDIR + " " + getProp(FEATUREDIR) + " does not exist; ");
+			if (!unitfeatureDir.mkdir()) {
+				throw new Error("Could not create FEATUREDIR");
+			}
+			System.out.print("Created successfully.\n");
+		}
+
+		maryInputType = "ALLOPHONES";
+		maryOutputType = "TARGETFEATURES";
+	}
+
+	public SortedMap<String, String> getDefaultProps(DatabaseLayout theDb) {
+		this.db = theDb;
+		if (props == null) {
+			props = new TreeMap<String, String>();
+			props.put(FEATUREDIR, db.getProp(db.ROOTDIR) + "phonefeatures" + System.getProperty("file.separator"));
+			props.put(ALLOPHONES, db.getProp(db.ROOTDIR) + "allophones" + System.getProperty("file.separator"));
+			props.put(FEATURELIST, db.getProp(db.CONFIGDIR) + "features.txt");
+			props.put(MARYSERVERHOST, "localhost");
+			props.put(MARYSERVERPORT, "59125");
+		}
+
+		return props;
+	}
+
+	protected void setupHelp() {
+		props2Help = new TreeMap<String, String>();
+		props2Help.put(FEATUREDIR, "directory containing the phone features." + "Will be created if it does not exist");
+		props2Help.put(ALLOPHONES, "Directory of corrected allophones files.");
+		props2Help.put(MARYSERVERHOST, "the host were the Mary server is running, default: \"localhost\"");
+		props2Help.put(MARYSERVERPORT, "the port were the Mary server is listening, default: \"59125\"");
+	}
+
+	public MaryHttpClient getMaryClient() throws IOException {
+		if (mary == null) {
+			try {
+				mary = new MaryHttpClient(new Address(getProp(MARYSERVERHOST), Integer.parseInt(getProp(MARYSERVERPORT))));
+			} catch (IOException e) {
+				throw new IOException("Could not connect to Maryserver at " + getProp(MARYSERVERHOST) + " "
+						+ getProp(MARYSERVERPORT));
+			}
+		}
+		return mary;
+	}
+
+	protected void loadFeatureList() throws IOException {
+		File featureFile = new File(getProp(FEATURELIST));
+		if (!featureFile.exists()) {
+			System.out.println("No feature file: '" + getProp(FEATURELIST) + "'");
+		} else {
+			System.out.println("Loading features from file " + getProp(FEATURELIST));
+			try {
+				featureList = FileUtils.getFileAsString(featureFile, "UTF-8");
+				featureList = featureList.replaceAll("\\s+", " ");
+				// Exclude specific halfphone features if present:
+				for (String f : HalfPhoneUnitFeatureComputer.HALFPHONE_FEATURES) {
+					if (featureList.contains(f)) {
+						featureList = featureList.replaceAll(f, "");
+					}
+				}
+				if (!featureList.contains(PHONEFEATURE)) {
+					throw new RuntimeException("Feature list does not contain feature '" + PHONEFEATURE
+							+ "'. It makes no sense to continue.");
+				}
+				if (!featureList.startsWith(PHONEFEATURE)) {
+					// PHONEFEATURE must be the first one in the list
+					featureList = featureList.replaceFirst("\\s+" + PHONEFEATURE + "\\s*", " ");
+					featureList = PHONEFEATURE + " " + featureList;
+				}
+			} catch (IOException e) {
+				IOException ioe = new IOException("Cannot read list of features");
+				ioe.initCause(e);
+				throw ioe;
+			}
+		}
+
+	}
+
+	public boolean compute() throws IOException {
+
+		loadFeatureList();
+
+		textDir = new File(db.getProp(db.TEXTDIR));
+		System.out.println("Computing unit features for " + bnl.getLength() + " files");
+		for (int i = 0; i < bnl.getLength(); i++) {
+			percent = 100 * i / bnl.getLength();
+			computeFeaturesFor(bnl.getName(i));
+			System.out.println("    " + bnl.getName(i));
+		}
+		System.out.println("Finished computing the unit features.");
+		return true;
+	}
+
+	public void computeFeaturesFor(String basename) throws IOException {
+		File allophoneFile = new File(getProp(ALLOPHONES) + basename + xmlExt);
+		String text = FileUtils.getFileAsString(allophoneFile, "UTF-8");
+
+		OutputStream os = new BufferedOutputStream(new FileOutputStream(new File(unitfeatureDir, basename + featsExt)));
+		MaryHttpClient maryClient = getMaryClient();
+
+		maryClient.process(text, maryInputType, maryOutputType, locale, null, null, "", null, featureList, os);
+		os.flush();
+		os.close();
+	}
+
+	/**
+	 * Provide the progress of computation, in percent, or -1 if that feature is not implemented.
+	 * 
+	 * @return -1 if not implemented, or an integer between 0 and 100.
+	 */
+	public int getProgress() {
+		return percent;
+	}
 
 }
-

marytts-builder/src/main/java/marytts/tools/voiceimport/PhoneUnitLabelComputer.java

@@ -35,234 +35,223 @@ import marytts.util.data.text.XwavesLabelfileReader;
 
 /**
  * Compute unit labels from phone labels.
+ * 
  * @author schroed
  *
  */
-public class PhoneUnitLabelComputer extends VoiceImportComponent
-{
-    protected File phonelabelDir;
-    protected File unitlabelDir;
-    protected String unitlabelExt;
-    protected String pauseSymbol;
-        
-    protected DatabaseLayout db = null;
-    protected int percent = 0;
-    protected int basenameIndex;
-    
-    public String getName(){
-        return "PhoneUnitLabelComputer";
-    }
-    
-    @Override
-    protected void initialiseComp()
-    throws Exception
-    {
-    	pauseSymbol = db.getAllophoneSet().getSilence().name();
+public class PhoneUnitLabelComputer extends VoiceImportComponent {
+	protected File phonelabelDir;
+	protected File unitlabelDir;
+	protected String unitlabelExt;
+	protected String pauseSymbol;
 
-        phonelabelDir = new File(db.getProp(DatabaseLayout.LABDIR));
-        unitlabelDir = new File(db.getProp(DatabaseLayout.PHONELABDIR));
-        unitlabelExt = db.getProp(DatabaseLayout.LABEXT);
-    }
-    
-     public SortedMap<String, String> getDefaultProps(DatabaseLayout db){
-        this.db = db;
-       if (props == null){
-           props = new TreeMap<String, String>();
-       }
-       return props;
-    }
-     
-    protected void setupHelp(){
-        props2Help = new TreeMap<String, String>();
-    } 
-    
-    public boolean compute() throws Exception
-    {
-        if (!phonelabelDir.exists()) {
-        	throw new IOException("No such directory: "+ phonelabelDir);
-        }
-        
-        if (!unitlabelDir.exists()) {
-            System.out.print(DatabaseLayout.PHONELABDIR+" "+db.getProp(DatabaseLayout.PHONELABDIR) + " does not exist; ");
-            if (!unitlabelDir.mkdir()) {
-                throw new Exception("Could not create PHONELABDIR");
-            }
-            System.out.print("Created successfully.\n");
-        }
-        
-        System.out.println( "Computing unit labels for " 
-                + bnl.getLength() + " files." );
-        System.out.println( "From phonetic label files: " 
-                + db.getProp(DatabaseLayout.LABDIR) + "*" + db.getProp(DatabaseLayout.LABEXT));
-        System.out.println( "To       unit label files: " 
-                + unitlabelDir + "*" + unitlabelExt );
-        for (basenameIndex=0; basenameIndex<bnl.getLength(); basenameIndex++) {
-            percent = 100*basenameIndex/bnl.getLength();
-            computePhoneLabel(bnl.getName(basenameIndex));
-        }
-        System.out.println("Finished computing unit labels");
-        return true;
-    }
-    
-    public void computePhoneLabel(String baseName) throws Exception{
-        File labFile = new File( db.getProp(DatabaseLayout.LABDIR), baseName + db.getProp(DatabaseLayout.LABEXT) );
-        if ( !labFile.exists() ) {
-            System.out.println( "Utterance [" + baseName + "] does not have a phonetic label file." );
-            System.out.println( "Removing this utterance from the base utterance list." );
-            bnl.remove( baseName );
-            basenameIndex--;
-            return;
-        }
+	protected DatabaseLayout db = null;
+	protected int percent = 0;
+	protected int basenameIndex;
 
-        // parse labFile:
-        XwavesLabelfileReader labFileData = new XwavesLabelfileReader(labFile.getPath());
-        ArrayList<Double> endTimes = new ArrayList<Double>(Arrays.asList(labFileData.getTimes()));
-        ArrayList<String> labels = new ArrayList<String>(Arrays.asList(labFileData.getLabelSymbols()));
+	public String getName() {
+		return "PhoneUnitLabelComputer";
+	}
 
-        // ensure that each labeled interval ends after the previous one:
-        ListIterator<Double> timeIterator = endTimes.listIterator();
-        double time = timeIterator.next();
-        while (timeIterator.hasNext()) {
-            double nextTime = timeIterator.next();
-            if (time == nextTime) {
-                int index = timeIterator.previousIndex() - 1;
-                String label = labels.get(index);
-                // doesn't matter which of the two times we remove -- they're the same:
-                timeIterator.remove();
-                // but from the labels, we remove the previous one: 
-                labels.remove(index);
-                System.err.format("WARNING: labeled interval %d (%s) has zero duration; deleting it!\n", index + 1, label);
-            } else if (time > nextTime) {
-                throw new Exception("ERROR: labeled intervals are out of order; please fix the label file!");
-            }            
-            time = nextTime;
-        }
+	@Override
+	protected void initialiseComp() throws Exception {
+		pauseSymbol = db.getAllophoneSet().getSilence().name();
 
-        // merge consecutive pauses:
-        ListIterator<String> labelIterator = labels.listIterator();
-        String label = labelIterator.next();
-        while (labelIterator.hasNext()) {
-            String nextLabel = labelIterator.next();
-            if (label.equals(nextLabel) && label.equals(pauseSymbol)) {
-                labelIterator.remove();
-                endTimes.remove(labelIterator.previousIndex());
-            }
-            label = nextLabel;
-        }
-        
-        // get midtimes:
-        List<Double> midTimes = getMidTimes(labels, endTimes);
-        
-        // convert labels to unit labels:
-        String[] unitLabelLines = toUnitLabels(labels, endTimes, midTimes);
+		phonelabelDir = new File(db.getProp(DatabaseLayout.LABDIR));
+		unitlabelDir = new File(db.getProp(DatabaseLayout.PHONELABDIR));
+		unitlabelExt = db.getProp(DatabaseLayout.LABEXT);
+	}
 
-        // write to phonelab file:
-        File phoneLabFile = new File(unitlabelDir, baseName + unitlabelExt);
-        PrintWriter out = new PrintWriter(phoneLabFile);
-        // header:
-        for (String headerLine : labFileData.getHeader()) {
-            out.println(headerLine);
-        }
-        out.println("format: end time, unit index, phone");
-        out.println("#");
-        // labels:
-        for (String unitLabelLine : unitLabelLines) {
-            out.println(unitLabelLine);
-        }
-        out.close();      
-    }
-    
-    /**
-     * Get mid points for an utterance, given a list its phone labels and a list of corresponding end points.
-     * 
-     * @param labels
-     *            of the phones
-     * @param endTimes
-     *            of the phones
-     * @return a list of midpoint times (in seconds) for the phones
-     */
-    protected List<Double> getMidTimes(List<String> labels, List<Double> endTimes) {
-        // in this class, we don't actually need any midpoint times, so return null:
-        return null;
-    }
+	public SortedMap<String, String> getDefaultProps(DatabaseLayout db) {
+		this.db = db;
+		if (props == null) {
+			props = new TreeMap<String, String>();
+		}
+		return props;
+	}
 
+	protected void setupHelp() {
+		props2Help = new TreeMap<String, String>();
+	}
 
-    /**
-     * Convert phone labels to unit labels. This base implementation
-     * returns the phone labels; subclasses may want to override that
-     * behaviour.
-     * @param phoneLabels the phone labels, one phone per line, with each
-     * line containing three fields: 1. the end time of the current phone,
-     * in seconds, since the beginning of the file; 2. a number to be ignored;
-     * 3. the phone symbol.
-     * @return an array of lines, in the same format as the phoneLabels input
-     * array, but with unit symbols instead of phone symbols. The
-     * number in the middle now denotes the unit index. This array may
-     * or may not have the same number of lines as phoneLabels.
-     */
-    @Deprecated
-    protected String[] toUnitLabels(String[] phoneLabels)
-    {
-        String[] unitLabels = new String[phoneLabels.length];
-        int unitIndex = 0;
-        for (int i=0;i<phoneLabels.length;i++){
-            String line = phoneLabels[i];
-            unitIndex++;
-            StringTokenizer st = new StringTokenizer(line.trim());
-            //first token is time
-            String time = st.nextToken();
-            //next token is some number, throw away
-            st.nextToken();
-            //next token is phone
-            String phone = st.nextToken();
-            unitLabels[i] = time+" "+unitIndex+" "+phone;
-        }
-        return unitLabels; 
-    }
-    
-    /**
-     * Generate a sequence of Strings, corresponding to the lines in an Xwaves-compatible label file, by interleaving a List of
-     * label Strings with a List of end time points.
-     * 
-     * @param labels
-     *            a List of label Strings
-     * @param endTimes
-     *            a List of time points representing the end points of these labels
-     * @param midTimes
-     *            a List of time points representing the mid points of these labels (can be null)
-     * @return the label files lines
-     */
-    protected String[] toUnitLabels(List<String> labels, List<Double> endTimes, List<Double> midTimes) {
-        assert labels.size() == endTimes.size();
-        if (midTimes != null) {
-            assert midTimes.size() == endTimes.size();
-        }
-        
-        ArrayList<String> unitLines = new ArrayList<String>(labels.size());
-        
-        for (int i = 0; i < labels.size(); i++) {
-            String label = labels.get(i);
-            double endTime = endTimes.get(i);
-            if (midTimes != null) {
-                double midTime = midTimes.get(i);
-                unitLines.add(String.format(Locale.US, "%f %d %s_L", midTime, unitLines.size() + 1, label));
-                unitLines.add(String.format(Locale.US, "%f %d %s_R", endTime, unitLines.size() + 1, label));
-            } else {
-                unitLines.add(String.format(Locale.US, "%f %d %s", endTime, unitLines.size() + 1, label));
-            }
-        }
-        return (String[]) unitLines.toArray(new String[unitLines.size()]);
-    }
-    
-    /**
-     * Provide the progress of computation, in percent, or -1 if
-     * that feature is not implemented.
-     * @return -1 if not implemented, or an integer between 0 and 100.
-     */
-    public int getProgress()
-    {
-        return percent;
-    }
+	public boolean compute() throws Exception {
+		if (!phonelabelDir.exists()) {
+			throw new IOException("No such directory: " + phonelabelDir);
+		}
+
+		if (!unitlabelDir.exists()) {
+			System.out.print(DatabaseLayout.PHONELABDIR + " " + db.getProp(DatabaseLayout.PHONELABDIR) + " does not exist; ");
+			if (!unitlabelDir.mkdir()) {
+				throw new Exception("Could not create PHONELABDIR");
+			}
+			System.out.print("Created successfully.\n");
+		}
+
+		System.out.println("Computing unit labels for " + bnl.getLength() + " files.");
+		System.out.println("From phonetic label files: " + db.getProp(DatabaseLayout.LABDIR) + "*"
+				+ db.getProp(DatabaseLayout.LABEXT));
+		System.out.println("To       unit label files: " + unitlabelDir + "*" + unitlabelExt);
+		for (basenameIndex = 0; basenameIndex < bnl.getLength(); basenameIndex++) {
+			percent = 100 * basenameIndex / bnl.getLength();
+			computePhoneLabel(bnl.getName(basenameIndex));
+		}
+		System.out.println("Finished computing unit labels");
+		return true;
+	}
+
+	public void computePhoneLabel(String baseName) throws Exception {
+		File labFile = new File(db.getProp(DatabaseLayout.LABDIR), baseName + db.getProp(DatabaseLayout.LABEXT));
+		if (!labFile.exists()) {
+			System.out.println("Utterance [" + baseName + "] does not have a phonetic label file.");
+			System.out.println("Removing this utterance from the base utterance list.");
+			bnl.remove(baseName);
+			basenameIndex--;
+			return;
+		}
+
+		// parse labFile:
+		XwavesLabelfileReader labFileData = new XwavesLabelfileReader(labFile.getPath());
+		ArrayList<Double> endTimes = new ArrayList<Double>(Arrays.asList(labFileData.getTimes()));
+		ArrayList<String> labels = new ArrayList<String>(Arrays.asList(labFileData.getLabelSymbols()));
+
+		// ensure that each labeled interval ends after the previous one:
+		ListIterator<Double> timeIterator = endTimes.listIterator();
+		double time = timeIterator.next();
+		while (timeIterator.hasNext()) {
+			double nextTime = timeIterator.next();
+			if (time == nextTime) {
+				int index = timeIterator.previousIndex() - 1;
+				String label = labels.get(index);
+				// doesn't matter which of the two times we remove -- they're the same:
+				timeIterator.remove();
+				// but from the labels, we remove the previous one:
+				labels.remove(index);
+				System.err.format("WARNING: labeled interval %d (%s) has zero duration; deleting it!\n", index + 1, label);
+			} else if (time > nextTime) {
+				throw new Exception("ERROR: labeled intervals are out of order; please fix the label file!");
+			}
+			time = nextTime;
+		}
+
+		// merge consecutive pauses:
+		ListIterator<String> labelIterator = labels.listIterator();
+		String label = labelIterator.next();
+		while (labelIterator.hasNext()) {
+			String nextLabel = labelIterator.next();
+			if (label.equals(nextLabel) && label.equals(pauseSymbol)) {
+				labelIterator.remove();
+				endTimes.remove(labelIterator.previousIndex());
+			}
+			label = nextLabel;
+		}
+
+		// get midtimes:
+		List<Double> midTimes = getMidTimes(labels, endTimes);
+
+		// convert labels to unit labels:
+		String[] unitLabelLines = toUnitLabels(labels, endTimes, midTimes);
+
+		// write to phonelab file:
+		File phoneLabFile = new File(unitlabelDir, baseName + unitlabelExt);
+		PrintWriter out = new PrintWriter(phoneLabFile);
+		// header:
+		for (String headerLine : labFileData.getHeader()) {
+			out.println(headerLine);
+		}
+		out.println("format: end time, unit index, phone");
+		out.println("#");
+		// labels:
+		for (String unitLabelLine : unitLabelLines) {
+			out.println(unitLabelLine);
+		}
+		out.close();
+	}
+
+	/**
+	 * Get mid points for an utterance, given a list its phone labels and a list of corresponding end points.
+	 * 
+	 * @param labels
+	 *            of the phones
+	 * @param endTimes
+	 *            of the phones
+	 * @return a list of midpoint times (in seconds) for the phones
+	 */
+	protected List<Double> getMidTimes(List<String> labels, List<Double> endTimes) {
+		// in this class, we don't actually need any midpoint times, so return null:
+		return null;
+	}
+
+	/**
+	 * Convert phone labels to unit labels. This base implementation returns the phone labels; subclasses may want to override
+	 * that behaviour.
+	 * 
+	 * @param phoneLabels
+	 *            the phone labels, one phone per line, with each line containing three fields: 1. the end time of the current
+	 *            phone, in seconds, since the beginning of the file; 2. a number to be ignored; 3. the phone symbol.
+	 * @return an array of lines, in the same format as the phoneLabels input array, but with unit symbols instead of phone
+	 *         symbols. The number in the middle now denotes the unit index. This array may or may not have the same number of
+	 *         lines as phoneLabels.
+	 */
+	@Deprecated
+	protected String[] toUnitLabels(String[] phoneLabels) {
+		String[] unitLabels = new String[phoneLabels.length];
+		int unitIndex = 0;
+		for (int i = 0; i < phoneLabels.length; i++) {
+			String line = phoneLabels[i];
+			unitIndex++;
+			StringTokenizer st = new StringTokenizer(line.trim());
+			// first token is time
+			String time = st.nextToken();
+			// next token is some number, throw away
+			st.nextToken();
+			// next token is phone
+			String phone = st.nextToken();
+			unitLabels[i] = time + " " + unitIndex + " " + phone;
+		}
+		return unitLabels;
+	}
+
+	/**
+	 * Generate a sequence of Strings, corresponding to the lines in an Xwaves-compatible label file, by interleaving a List of
+	 * label Strings with a List of end time points.
+	 * 
+	 * @param labels
+	 *            a List of label Strings
+	 * @param endTimes
+	 *            a List of time points representing the end points of these labels
+	 * @param midTimes
+	 *            a List of time points representing the mid points of these labels (can be null)
+	 * @return the label files lines
+	 */
+	protected String[] toUnitLabels(List<String> labels, List<Double> endTimes, List<Double> midTimes) {
+		assert labels.size() == endTimes.size();
+		if (midTimes != null) {
+			assert midTimes.size() == endTimes.size();
+		}
+
+		ArrayList<String> unitLines = new ArrayList<String>(labels.size());
+
+		for (int i = 0; i < labels.size(); i++) {
+			String label = labels.get(i);
+			double endTime = endTimes.get(i);
+			if (midTimes != null) {
+				double midTime = midTimes.get(i);
+				unitLines.add(String.format(Locale.US, "%f %d %s_L", midTime, unitLines.size() + 1, label));
+				unitLines.add(String.format(Locale.US, "%f %d %s_R", endTime, unitLines.size() + 1, label));
+			} else {
+				unitLines.add(String.format(Locale.US, "%f %d %s", endTime, unitLines.size() + 1, label));
+			}
+		}
+		return (String[]) unitLines.toArray(new String[unitLines.size()]);
+	}
+
+	/**
+	 * Provide the progress of computation, in percent, or -1 if that feature is not implemented.
+	 * 
+	 * @return -1 if not implemented, or an integer between 0 and 100.
+	 */
+	public int getProgress() {
+		return percent;
+	}
 
 }
-

marytts-builder/src/main/java/marytts/tools/voiceimport/PhoneUnitfileWriter.java

@@ -34,169 +34,169 @@ import marytts.unitselection.data.UnitFileReader;
 import marytts.util.data.ESTTrackReader;
 import marytts.util.data.MaryHeader;
 
+public class PhoneUnitfileWriter extends VoiceImportComponent {
+	protected File maryDir;
+	protected String unitFileName;
+	protected File unitlabelDir;
+	protected int samplingRate;
+	protected String pauseSymbol;
+	protected PhoneLabelFeatureAligner aligner;
 
-public class PhoneUnitfileWriter extends VoiceImportComponent
-{
-    protected File maryDir;
-    protected String unitFileName;
-    protected File unitlabelDir;
-    protected int samplingRate;
-    protected String pauseSymbol;
-    protected PhoneLabelFeatureAligner aligner;
-    
-    protected String unitlabelExt = ".lab";
+	protected String unitlabelExt = ".lab";
 
-    protected DatabaseLayout db = null;
-    protected int percent = 0;
-    
-    public String UNITFILE = "PhoneUnitfileWriter.unitFile";
-    
-    public String getName(){
-        return "PhoneUnitfileWriter";
-    }
-    
-    @Override
-    protected void initialiseComp()
-    throws Exception
-    {
-        maryDir = new File(db.getProp(DatabaseLayout.FILEDIR));
-        
-        samplingRate = Integer.parseInt(db.getProp(DatabaseLayout.SAMPLINGRATE));
-        pauseSymbol = db.getAllophoneSet().getSilence().name();
-    
-        unitFileName = getProp(UNITFILE);
-        unitlabelDir = new File(db.getProp(DatabaseLayout.PHONELABDIR));
-        aligner = new PhoneLabelFeatureAligner();
-        db.initialiseComponent(aligner);        
-    }
-    
-    public SortedMap<String, String> getDefaultProps(DatabaseLayout db){
-        this.db = db;
-       if (props == null){
-           props = new TreeMap<String, String>();
-           props.put(UNITFILE, db.getProp(DatabaseLayout.FILEDIR)
-                   +"phoneUnits"+db.getProp(DatabaseLayout.MARYEXT));           
-       }
-       return props;
-    }
-    
-    protected void setupHelp(){
-        props2Help = new TreeMap<String, String>();
-        props2Help.put(UNITFILE, "file containing all phone units. Will be created by this module");           
-    }
-    
-    public boolean compute() throws Exception
-    {
-        
-        System.out.println("Unitfile writer started.");
-        System.out.println("Verifying that unit feature and label files are perfectly aligned...");
-        
-        if (!aligner.compute()) throw new IllegalStateException("Database is NOT perfectly aligned. Cannot create unit file.");
-        System.out.println("OK, alignment verified.");
-        DataOutputStream out = new DataOutputStream(new BufferedOutputStream(new FileOutputStream(unitFileName)));
-        long posNumUnits = new MaryHeader(MaryHeader.UNITS).writeTo(out);
-        out.writeInt(-1); // number of units; needs to be corrected later.
-        out.writeInt(samplingRate);
-        
-        // Loop over all utterances
-        int index = 0; // the unique index number of units in the unit file
-        long globalStart = 0l; // time, given as sample position with samplingRate
-        long localStart = 0l; // time, given as sample position with samplingRate
-        long totalNbrSamples = 0l;
-        long localNbrSamples = 0l;
-        long localNbrSamplesFromPM = 0l;
-        ESTTrackReader pmFile = null;
-        for (int i=0; i<bnl.getLength(); i++) {
-            percent = 100*i/bnl.getLength();
-            /* Open the relevant pitchmark file */
-            pmFile = new ESTTrackReader(db.getProp(DatabaseLayout.PMDIR)
-                    + bnl.getName(i) + db.getProp(DatabaseLayout.PMEXT));
-            // Output the utterance start marker: "null" unit
-            out.writeLong( globalStart ); out.writeInt(-1);
-            index++;
-            // Open the label file and reset the local time pointer
-            BufferedReader labels = new BufferedReader(new InputStreamReader(new FileInputStream(new File(unitlabelDir, bnl.getName(i) + unitlabelExt)), "UTF-8"));
-            String line;
-            localNbrSamples = 0l;
-            localStart = 0l;
-            // Skip label file header
-            while ((line = labels.readLine()) != null) {
-                if (line.startsWith("#")) break; // line starting with "#" marks end of header
-            }
-            // Now read the actual units
-            System.out.println("Start unit listing in file " + bnl.getName(i));
-            while ((line = labels.readLine()) != null) {
-                // Get the line and the double value in first position
-                line = line.trim();
-                if (line.equals("")) continue; // ignore empty lines
-                String[] parts = line.split( "\\s", 3 );
-                String unit_name=parts[2];
-                double endTime = Double.valueOf(parts[0]).doubleValue();
-                /* Relocate the label-specific end time on the nearest pitchmark */
-                endTime = (double)( pmFile.getClosestTime( endTime ) );
-                long end = (long)( endTime * (double)(samplingRate) );
-                // Output the unit
-                int duration = (int) (end - localStart);
-                out.writeLong( globalStart + localStart ); out.writeInt(duration);
-                //System.out.println( "Unit [" + index + "] starts at [" + localStart + "] and has duration [" + duration + "]." );
-                System.out.println( "Unit [" + index + "] <"+ unit_name + "> starts at [" + (double)localStart/(samplingRate) + "] and has duration [" + (double)duration/(samplingRate) + "]." );
-                // Update various pointers
-                localStart = end;
-                index++;
-                localNbrSamples += duration;
-                totalNbrSamples += duration;
-            }
-            // Output the utterance end marker: "null" unit
-            out.writeLong( globalStart + localStart ); out.writeInt(-1);
-            index++;
-            /* Locate the global start of the next file:
-             * this corrects the discrepancy between the duration of the label file and the duration
-             * of the pitchmark file (which is considered as the authority). */
-            localNbrSamplesFromPM = (long)( (double)( pmFile.getTimeSpan() ) * (double)(samplingRate) );
-            globalStart += localNbrSamplesFromPM;
-            /* Clean the house */
-            labels.close();
-            System.out.println( "    " + bnl.getName(i) + " (" + index + ") (This file has [" + localNbrSamples
-                    + "] samples from .lab, rectified to [" + localNbrSamplesFromPM
-                    + "] from the pitchmarks, diff [" + (localNbrSamplesFromPM - localNbrSamples) + "], cumul [" + globalStart + "])" );
-            if ( (localNbrSamplesFromPM - localNbrSamples) < 0 ) System.out.println( "BORK BORK BORK: .lab file longer than pitchmarks !" );
-        }
-        out.close();
-        // Now index is the number of units. Set this in the file:
-        RandomAccessFile raf = new RandomAccessFile(unitFileName, "rw");
-        raf.seek(posNumUnits);
-        raf.writeInt(index);
-        raf.close();
-        System.out.println("Number of processed units: " + index );
+	protected DatabaseLayout db = null;
+	protected int percent = 0;
 
-        
-        UnitFileReader tester = new UnitFileReader(unitFileName);
-        int unitsOnDisk = tester.getNumberOfUnits();
-        if (unitsOnDisk == index) {
-            System.out.println("Can read right number of units");
-            return true;
-        } else {
-            System.out.println("Read wrong number of units: "+unitsOnDisk);
-            return false;
-        }
-    }
+	public String UNITFILE = "PhoneUnitfileWriter.unitFile";
 
-    /**
-     * Provide the progress of computation, in percent, or -1 if
-     * that feature is not implemented.
-     * @return -1 if not implemented, or an integer between 0 and 100.
-     */
-    public int getProgress()
-    {
-        return percent;
-    }
+	public String getName() {
+		return "PhoneUnitfileWriter";
+	}
 
-    public static void main(String[] args) throws Exception
-    {
-        PhoneUnitfileWriter ufw = new PhoneUnitfileWriter();
-        new DatabaseLayout(ufw); 
-        ufw.compute();
-    }
+	@Override
+	protected void initialiseComp() throws Exception {
+		maryDir = new File(db.getProp(DatabaseLayout.FILEDIR));
+
+		samplingRate = Integer.parseInt(db.getProp(DatabaseLayout.SAMPLINGRATE));
+		pauseSymbol = db.getAllophoneSet().getSilence().name();
+
+		unitFileName = getProp(UNITFILE);
+		unitlabelDir = new File(db.getProp(DatabaseLayout.PHONELABDIR));
+		aligner = new PhoneLabelFeatureAligner();
+		db.initialiseComponent(aligner);
+	}
+
+	public SortedMap<String, String> getDefaultProps(DatabaseLayout db) {
+		this.db = db;
+		if (props == null) {
+			props = new TreeMap<String, String>();
+			props.put(UNITFILE, db.getProp(DatabaseLayout.FILEDIR) + "phoneUnits" + db.getProp(DatabaseLayout.MARYEXT));
+		}
+		return props;
+	}
+
+	protected void setupHelp() {
+		props2Help = new TreeMap<String, String>();
+		props2Help.put(UNITFILE, "file containing all phone units. Will be created by this module");
+	}
+
+	public boolean compute() throws Exception {
+
+		System.out.println("Unitfile writer started.");
+		System.out.println("Verifying that unit feature and label files are perfectly aligned...");
+
+		if (!aligner.compute())
+			throw new IllegalStateException("Database is NOT perfectly aligned. Cannot create unit file.");
+		System.out.println("OK, alignment verified.");
+		DataOutputStream out = new DataOutputStream(new BufferedOutputStream(new FileOutputStream(unitFileName)));
+		long posNumUnits = new MaryHeader(MaryHeader.UNITS).writeTo(out);
+		out.writeInt(-1); // number of units; needs to be corrected later.
+		out.writeInt(samplingRate);
+
+		// Loop over all utterances
+		int index = 0; // the unique index number of units in the unit file
+		long globalStart = 0l; // time, given as sample position with samplingRate
+		long localStart = 0l; // time, given as sample position with samplingRate
+		long totalNbrSamples = 0l;
+		long localNbrSamples = 0l;
+		long localNbrSamplesFromPM = 0l;
+		ESTTrackReader pmFile = null;
+		for (int i = 0; i < bnl.getLength(); i++) {
+			percent = 100 * i / bnl.getLength();
+			/* Open the relevant pitchmark file */
+			pmFile = new ESTTrackReader(db.getProp(DatabaseLayout.PMDIR) + bnl.getName(i) + db.getProp(DatabaseLayout.PMEXT));
+			// Output the utterance start marker: "null" unit
+			out.writeLong(globalStart);
+			out.writeInt(-1);
+			index++;
+			// Open the label file and reset the local time pointer
+			BufferedReader labels = new BufferedReader(new InputStreamReader(new FileInputStream(new File(unitlabelDir,
+					bnl.getName(i) + unitlabelExt)), "UTF-8"));
+			String line;
+			localNbrSamples = 0l;
+			localStart = 0l;
+			// Skip label file header
+			while ((line = labels.readLine()) != null) {
+				if (line.startsWith("#"))
+					break; // line starting with "#" marks end of header
+			}
+			// Now read the actual units
+			System.out.println("Start unit listing in file " + bnl.getName(i));
+			while ((line = labels.readLine()) != null) {
+				// Get the line and the double value in first position
+				line = line.trim();
+				if (line.equals(""))
+					continue; // ignore empty lines
+				String[] parts = line.split("\\s", 3);
+				String unit_name = parts[2];
+				double endTime = Double.valueOf(parts[0]).doubleValue();
+				/* Relocate the label-specific end time on the nearest pitchmark */
+				endTime = (double) (pmFile.getClosestTime(endTime));
+				long end = (long) (endTime * (double) (samplingRate));
+				// Output the unit
+				int duration = (int) (end - localStart);
+				out.writeLong(globalStart + localStart);
+				out.writeInt(duration);
+				// System.out.println( "Unit [" + index + "] starts at [" + localStart + "] and has duration [" + duration + "]."
+				// );
+				System.out.println("Unit [" + index + "] <" + unit_name + "> starts at [" + (double) localStart / (samplingRate)
+						+ "] and has duration [" + (double) duration / (samplingRate) + "].");
+				// Update various pointers
+				localStart = end;
+				index++;
+				localNbrSamples += duration;
+				totalNbrSamples += duration;
+			}
+			// Output the utterance end marker: "null" unit
+			out.writeLong(globalStart + localStart);
+			out.writeInt(-1);
+			index++;
+			/*
+			 * Locate the global start of the next file: this corrects the discrepancy between the duration of the label file and
+			 * the duration of the pitchmark file (which is considered as the authority).
+			 */
+			localNbrSamplesFromPM = (long) ((double) (pmFile.getTimeSpan()) * (double) (samplingRate));
+			globalStart += localNbrSamplesFromPM;
+			/* Clean the house */
+			labels.close();
+			System.out.println("    " + bnl.getName(i) + " (" + index + ") (This file has [" + localNbrSamples
+					+ "] samples from .lab, rectified to [" + localNbrSamplesFromPM + "] from the pitchmarks, diff ["
+					+ (localNbrSamplesFromPM - localNbrSamples) + "], cumul [" + globalStart + "])");
+			if ((localNbrSamplesFromPM - localNbrSamples) < 0)
+				System.out.println("BORK BORK BORK: .lab file longer than pitchmarks !");
+		}
+		out.close();
+		// Now index is the number of units. Set this in the file:
+		RandomAccessFile raf = new RandomAccessFile(unitFileName, "rw");
+		raf.seek(posNumUnits);
+		raf.writeInt(index);
+		raf.close();
+		System.out.println("Number of processed units: " + index);
+
+		UnitFileReader tester = new UnitFileReader(unitFileName);
+		int unitsOnDisk = tester.getNumberOfUnits();
+		if (unitsOnDisk == index) {
+			System.out.println("Can read right number of units");
+			return true;
+		} else {
+			System.out.println("Read wrong number of units: " + unitsOnDisk);
+			return false;
+		}
+	}
+
+	/**
+	 * Provide the progress of computation, in percent, or -1 if that feature is not implemented.
+	 * 
+	 * @return -1 if not implemented, or an integer between 0 and 100.
+	 */
+	public int getProgress() {
+		return percent;
+	}
+
+	public static void main(String[] args) throws Exception {
+		PhoneUnitfileWriter ufw = new PhoneUnitfileWriter();
+		new DatabaseLayout(ufw);
+		ufw.compute();
+	}
 
 }
-

marytts-builder/src/main/java/marytts/tools/voiceimport/PraatPitchmarker.java

@@ -36,273 +36,272 @@ import marytts.util.math.ArrayUtils;
 import org.apache.commons.io.IOUtils;
 import org.apache.commons.lang.StringUtils;
 
+public class PraatPitchmarker extends VoiceImportComponent {
+	protected DatabaseLayout db = null;
+	protected String pointpExt = ".PointProcess";
+	protected String tmpScript;
 
-public class PraatPitchmarker extends VoiceImportComponent
-{
-    protected DatabaseLayout db = null;
-    protected String pointpExt = ".PointProcess";
-    protected String tmpScript;
-    
-    protected int percent = 0;
-    
-    public final String COMMAND     = getName()+".command";
-    public final String MINPITCH    = getName()+".minPitch";
-    public final String MAXPITCH    = getName()+".maxPitch";
-    public final String PRAATPMDIR  = getName()+".pmDir";
-    public final String WAVEDIR     = getName()+".waveDir";
-    
-    public final String PMDIR = "db.pmDir";
-    public final String PMEXT = "db.pmExtension";
+	protected int percent = 0;
 
-    protected void setupHelp()
-    {
-        if (props2Help ==null){
-            props2Help = new TreeMap();
-            props2Help.put(COMMAND, "The command that is used to launch praat");
-            props2Help.put(MINPITCH,"minimum value for the pitch (in Hz). Default: female 100, male 75");
-            props2Help.put(MAXPITCH,"maximum value for the pitch (in Hz). Default: female 500, male 300");            
-        }
-        
+	public final String COMMAND = getName() + ".command";
+	public final String MINPITCH = getName() + ".minPitch";
+	public final String MAXPITCH = getName() + ".maxPitch";
+	public final String PRAATPMDIR = getName() + ".pmDir";
+	public final String WAVEDIR = getName() + ".waveDir";
 
-    }
-    
-    public String getName(){
-        return "PraatPitchmarker";
-    }
-    
-    @Override
-    protected void initialiseComp()
-    {
-        tmpScript = db.getProp(db.TEMPDIR)+"script.praat";
-    }
-    
-    public SortedMap getDefaultProps(DatabaseLayout db){
-        this.db = db;
-       if (props == null){
-           props = new TreeMap();
-           props.put(COMMAND,"praat");   
-           if (db.getProp(db.GENDER).equals("female")){
-               props.put(MINPITCH,"100");
-               props.put(MAXPITCH,"500");
-           } else {
-               props.put(MINPITCH,"75");
-               props.put(MAXPITCH,"300");
-           }
-           props.put(WAVEDIR,db.getProp(db.WAVDIR));
-           props.put(PRAATPMDIR,db.getProp(db.PMDIR));
-           String rootDir = db.getProp(db.ROOTDIR);
-       }
-       return props;
-    }
-    
-    /**
-     * Shift the pitchmarks to the closest peak.
-     */
-    private float[] shiftToClosestPeak( float[] pmIn, short[] w, int sampleRate ) {
-        
-        final int HORIZON = 32; // <= number of samples to seek before and after the pitchmark
-        float[] pmOut = new float[pmIn.length];
-        
-        /* Browse the pitchmarks */
-        int pm = 0;
-        int pmwmax = w.length - 1;
-        int TO = 0;
-        int max = 0;
-        for ( int pi = 0; pi < pmIn.length; pi++ ) {
-            pm = (int)( pmIn[pi] * sampleRate );
-            // If the pitchmark goes out of the waveform (this sometimes
-            // happens with the last one due to rounding errors), just clip it.
-            if ( pm > pmwmax ) {
-                // If this was not the last pitchmark, there is a problem
-                if ( pi < (pmIn.length-1)) {
-                    throw new RuntimeException( "Some pitchmarks are located above the location of the last waveform sample !" );
-                }
-                // Else, if it was the last pitchmark, clip it:
-                pmOut[pi] = (float) ((double)(pmwmax) / (double)(sampleRate));
-            }
-            // Else, if the pitchmark is in the waveform:
-            else {
-                /* Seek the max of the wav samples around the pitchmark */
-                max = pm;
-                // - Back:
-                TO = (pm-HORIZON) < 0 ? 0 : (pm-HORIZON);
-                for ( int i = pm-1; i >= TO; i-- ) {
-                    if ( w[i] > w[max] ) max = i;
-                }
-                // - Forth:
-                TO = (pm+HORIZON+1) > w.length ? w.length : (pm+HORIZON+1);
-                for ( int i = pm+1; i < TO; i++ ) {
-                    if ( w[i] >= w[max] ) max = i;
-                }
-                /* Translate the pitchmark */
-                pmOut[pi] = (float) ( (double)(max) / (double)(sampleRate) );
-            }
-        }
-        
-        return pmOut;
-    }
-    
-    /**
-     * Shift the pitchmarks to the previous zero crossing.
-     */
-    private float[] shiftToPreviousZero( float[] pmIn, short[] w, int sampleRate ) {
-        
-        final int HORIZON = 32; // <= number of samples to seek before the pitchmark
-        float[] pmOut = new float[pmIn.length];
-        
-        /* Browse the pitchmarks */
-        int pm = 0;
-        int TO = 0;
-        int zero = 0;
-        for ( int pi = 0; pi < pmIn.length; pi++ ) {
-            pm = (int)( pmIn[pi] * sampleRate );
-            /* If the initial pitchmark is on a zero, don't shift the pitchmark. */
-            if ( w[pm] == 0 ) {
-                pmOut[pi] = pmIn[pi];
-                continue;
-            }
-            /* Else: */
-            /* Seek the zero crossing preceding the pitchmark */
-            TO = (pm-HORIZON) < 0 ? 0 : (pm-HORIZON);
-            for ( zero = pm; ( zero > TO ) && ( (w[zero]*w[zero+1]) > 0 ); zero-- );
-            /* If no zero crossing was found, don't move the pitchmark */
-            if ( (zero == TO) && ( (w[zero]*w[zero+1]) > 0 )  ) {
-                pmOut[pi] = pmIn[pi];
-            }
-            /* If a zero crossing was found, translate the pitchmark */
-            else {
-                pmOut[pi] = (float) ( (double)( (-w[zero]) <  w[zero+1] ?  zero : (zero+1) ) / (double)(sampleRate) );
-            }
-        }
-        
-        return pmOut;
-    }
-    
-    /**
-     * Adjust pitchmark position to the zero crossing preceding the closest peak.
-     * @param basename basename of the corresponding wav file
-     * @param pitchmarks the input pitchmarks
-     * @return the adjusted pitchmarks
-     */
-    private float[] adjustPitchmarks( String basename, float[] pitchmarks ) throws IOException
-    {
-        /* Load the wav file */
-        String fName = getProp(WAVEDIR) + basename + db.getProp(db.WAVEXT);
-        WavReader wf = new WavReader( fName );
-        short[] w = wf.getSamples();
-        float[] pmOut = null;
-        try {
-            /* Shift to the closest peak */
-            pmOut = shiftToClosestPeak( pitchmarks, w, wf.getSampleRate() );
-            /* Shift to the zero immediately preceding the closest peak */
-            pmOut = shiftToPreviousZero( pmOut, w, wf.getSampleRate() );
-        } catch ( RuntimeException e ) {
-            throw new RuntimeException( "For utterance [" + basename + "]:" , e );
-        }
-        return pmOut;
-    }
-    
-    @Deprecated
-    protected boolean praatPitchmarks(String basename) throws IOException
-    {
-        String wavFilename = new File(getProp(WAVEDIR) + basename + db.getProp(db.WAVEXT)).getAbsolutePath();
-        String pointprocessFilename = getProp(PRAATPMDIR)+basename+pointpExt;
+	public final String PMDIR = "db.pmDir";
+	public final String PMEXT = "db.pmExtension";
 
-        String strTmp = getProp(COMMAND)+" "+tmpScript+" "+wavFilename+" "+pointprocessFilename+" "+getProp(MINPITCH)+" "+getProp(MAXPITCH);
-        
-        if (MaryUtils.isWindows())
-            strTmp = "cmd.exe /c " + strTmp;
-        
-        General.launchProc(strTmp, "PraatPitchmarker", basename);
+	protected void setupHelp() {
+		if (props2Help == null) {
+			props2Help = new TreeMap();
+			props2Help.put(COMMAND, "The command that is used to launch praat");
+			props2Help.put(MINPITCH, "minimum value for the pitch (in Hz). Default: female 100, male 75");
+			props2Help.put(MAXPITCH, "maximum value for the pitch (in Hz). Default: female 500, male 300");
+		}
 
-        // Now convert the praat format into EST pm format:
-        estPitchmarks(basename);
+	}
 
-        return true;
-    }
-    
-    /**
-     * Convert Praat PointProcess files to EST pm files
-     * 
-     * @param basename of files to process
-     * @throws IOException
-     */
-    protected void estPitchmarks(String basename) throws IOException {
-        String pointprocessFilename = getProp(PRAATPMDIR) + basename + pointpExt;
-        String pmFilename = getProp(PRAATPMDIR) + basename + db.getProp(PMEXT);
-        FileReader pmReader = new FileReader(pointprocessFilename);
-        double[] pm = new PraatTextfileDoubleDataSource(pmReader).getAllData();
-        pmReader.close();
-        float[] pitchmarks = ArrayUtils.copyDouble2Float(pm);
-        new ESTTrackWriter(pitchmarks, null, "pitchmarks").doWriteAndClose(pmFilename, false, false);
-    }
-    
-    /**
-     * The standard compute() method of the VoiceImportComponent interface.
-     */
-    public boolean compute() throws IOException {
-        percent = 0;
-        
-        String[] baseNameArray = bnl.getListAsArray();
-        System.out.println( "Computing pitchmarks for " + baseNameArray.length + " utterances." );
+	public String getName() {
+		return "PraatPitchmarker";
+	}
 
-        /* Ensure the existence of the target pitchmark directory */
-        File dir = new File(getProp(PRAATPMDIR));
-        if (!dir.exists()) {
-            System.out.println( "Creating the directory [" + getProp(PRAATPMDIR) + "]." );
-            dir.mkdir();
-        }
-        
-        // script.praat is provided as template.
-        InputStream scriptResource = getClass().getResourceAsStream("script.praat");
-        FileWriter scriptWriter = new FileWriter(tmpScript);
-        IOUtils.copy(scriptResource, scriptWriter);
-        scriptWriter.close();
-        
-        // ensure that basenames.lst is up to date:
-        String baseNameListFileName = db.getProp("db.basenameFile");
-        bnl.write(baseNameListFileName);
+	@Override
+	protected void initialiseComp() {
+		tmpScript = db.getProp(db.TEMPDIR) + "script.praat";
+	}
 
-        // run Praat process:
-        String praatCommand = StringUtils.join(new String[] { getProp(COMMAND), tmpScript, baseNameListFileName,
-                getProp(WAVEDIR), getProp(PRAATPMDIR), getProp(MINPITCH), getProp(MAXPITCH) }, " ");
-        System.out.println("Running Praat as: " + praatCommand);
-        if (MaryUtils.isWindows()) {
-            praatCommand = "cmd.exe /c " + praatCommand;
-        }
-        General.launchProc(praatCommand, getName(), "");
-        
-        // convert to EST format
-        for ( int i = 0; i < baseNameArray.length; i++ ) {
-            percent = 100*i/baseNameArray.length;
-            estPitchmarks(baseNameArray[i]);
-        }
+	public SortedMap getDefaultProps(DatabaseLayout db) {
+		this.db = db;
+		if (props == null) {
+			props = new TreeMap();
+			props.put(COMMAND, "praat");
+			if (db.getProp(db.GENDER).equals("female")) {
+				props.put(MINPITCH, "100");
+				props.put(MAXPITCH, "500");
+			} else {
+				props.put(MINPITCH, "75");
+				props.put(MAXPITCH, "300");
+			}
+			props.put(WAVEDIR, db.getProp(db.WAVDIR));
+			props.put(PRAATPMDIR, db.getProp(db.PMDIR));
+			String rootDir = db.getProp(db.ROOTDIR);
+		}
+		return props;
+	}
 
-        return true;
-    }
-    
-    /**
-     * Provide the progress of computation, in percent, or -1 if
-     * that feature is not implemented.
-     * @return -1 if not implemented, or an integer between 0 and 100.
-     */
-    public int getProgress()
-    {
-        // hack to get progress from Praat through file I/O:
-        File percentFile = new File(db.getProp(db.TEMPDIR) + "percent");
-        if (percentFile.exists()) {
-            percent = -1;
-            try {
-                FileReader percentReader = new FileReader(percentFile);
-                String percentString = IOUtils.toString(percentReader);
-                percent = Integer.parseInt(percentString);
-                percentReader.close();
-            } catch (IOException e) {
-                // TODO Auto-generated catch block
-                e.printStackTrace();
-            }
-        }
-        return percent;
-    }
+	/**
+	 * Shift the pitchmarks to the closest peak.
+	 */
+	private float[] shiftToClosestPeak(float[] pmIn, short[] w, int sampleRate) {
+
+		final int HORIZON = 32; // <= number of samples to seek before and after the pitchmark
+		float[] pmOut = new float[pmIn.length];
+
+		/* Browse the pitchmarks */
+		int pm = 0;
+		int pmwmax = w.length - 1;
+		int TO = 0;
+		int max = 0;
+		for (int pi = 0; pi < pmIn.length; pi++) {
+			pm = (int) (pmIn[pi] * sampleRate);
+			// If the pitchmark goes out of the waveform (this sometimes
+			// happens with the last one due to rounding errors), just clip it.
+			if (pm > pmwmax) {
+				// If this was not the last pitchmark, there is a problem
+				if (pi < (pmIn.length - 1)) {
+					throw new RuntimeException("Some pitchmarks are located above the location of the last waveform sample !");
+				}
+				// Else, if it was the last pitchmark, clip it:
+				pmOut[pi] = (float) ((double) (pmwmax) / (double) (sampleRate));
+			}
+			// Else, if the pitchmark is in the waveform:
+			else {
+				/* Seek the max of the wav samples around the pitchmark */
+				max = pm;
+				// - Back:
+				TO = (pm - HORIZON) < 0 ? 0 : (pm - HORIZON);
+				for (int i = pm - 1; i >= TO; i--) {
+					if (w[i] > w[max])
+						max = i;
+				}
+				// - Forth:
+				TO = (pm + HORIZON + 1) > w.length ? w.length : (pm + HORIZON + 1);
+				for (int i = pm + 1; i < TO; i++) {
+					if (w[i] >= w[max])
+						max = i;
+				}
+				/* Translate the pitchmark */
+				pmOut[pi] = (float) ((double) (max) / (double) (sampleRate));
+			}
+		}
+
+		return pmOut;
+	}
+
+	/**
+	 * Shift the pitchmarks to the previous zero crossing.
+	 */
+	private float[] shiftToPreviousZero(float[] pmIn, short[] w, int sampleRate) {
+
+		final int HORIZON = 32; // <= number of samples to seek before the pitchmark
+		float[] pmOut = new float[pmIn.length];
+
+		/* Browse the pitchmarks */
+		int pm = 0;
+		int TO = 0;
+		int zero = 0;
+		for (int pi = 0; pi < pmIn.length; pi++) {
+			pm = (int) (pmIn[pi] * sampleRate);
+			/* If the initial pitchmark is on a zero, don't shift the pitchmark. */
+			if (w[pm] == 0) {
+				pmOut[pi] = pmIn[pi];
+				continue;
+			}
+			/* Else: */
+			/* Seek the zero crossing preceding the pitchmark */
+			TO = (pm - HORIZON) < 0 ? 0 : (pm - HORIZON);
+			for (zero = pm; (zero > TO) && ((w[zero] * w[zero + 1]) > 0); zero--)
+				;
+			/* If no zero crossing was found, don't move the pitchmark */
+			if ((zero == TO) && ((w[zero] * w[zero + 1]) > 0)) {
+				pmOut[pi] = pmIn[pi];
+			}
+			/* If a zero crossing was found, translate the pitchmark */
+			else {
+				pmOut[pi] = (float) ((double) ((-w[zero]) < w[zero + 1] ? zero : (zero + 1)) / (double) (sampleRate));
+			}
+		}
+
+		return pmOut;
+	}
+
+	/**
+	 * Adjust pitchmark position to the zero crossing preceding the closest peak.
+	 * 
+	 * @param basename
+	 *            basename of the corresponding wav file
+	 * @param pitchmarks
+	 *            the input pitchmarks
+	 * @return the adjusted pitchmarks
+	 */
+	private float[] adjustPitchmarks(String basename, float[] pitchmarks) throws IOException {
+		/* Load the wav file */
+		String fName = getProp(WAVEDIR) + basename + db.getProp(db.WAVEXT);
+		WavReader wf = new WavReader(fName);
+		short[] w = wf.getSamples();
+		float[] pmOut = null;
+		try {
+			/* Shift to the closest peak */
+			pmOut = shiftToClosestPeak(pitchmarks, w, wf.getSampleRate());
+			/* Shift to the zero immediately preceding the closest peak */
+			pmOut = shiftToPreviousZero(pmOut, w, wf.getSampleRate());
+		} catch (RuntimeException e) {
+			throw new RuntimeException("For utterance [" + basename + "]:", e);
+		}
+		return pmOut;
+	}
+
+	@Deprecated
+	protected boolean praatPitchmarks(String basename) throws IOException {
+		String wavFilename = new File(getProp(WAVEDIR) + basename + db.getProp(db.WAVEXT)).getAbsolutePath();
+		String pointprocessFilename = getProp(PRAATPMDIR) + basename + pointpExt;
+
+		String strTmp = getProp(COMMAND) + " " + tmpScript + " " + wavFilename + " " + pointprocessFilename + " "
+				+ getProp(MINPITCH) + " " + getProp(MAXPITCH);
+
+		if (MaryUtils.isWindows())
+			strTmp = "cmd.exe /c " + strTmp;
+
+		General.launchProc(strTmp, "PraatPitchmarker", basename);
+
+		// Now convert the praat format into EST pm format:
+		estPitchmarks(basename);
+
+		return true;
+	}
+
+	/**
+	 * Convert Praat PointProcess files to EST pm files
+	 * 
+	 * @param basename
+	 *            of files to process
+	 * @throws IOException
+	 */
+	protected void estPitchmarks(String basename) throws IOException {
+		String pointprocessFilename = getProp(PRAATPMDIR) + basename + pointpExt;
+		String pmFilename = getProp(PRAATPMDIR) + basename + db.getProp(PMEXT);
+		FileReader pmReader = new FileReader(pointprocessFilename);
+		double[] pm = new PraatTextfileDoubleDataSource(pmReader).getAllData();
+		pmReader.close();
+		float[] pitchmarks = ArrayUtils.copyDouble2Float(pm);
+		new ESTTrackWriter(pitchmarks, null, "pitchmarks").doWriteAndClose(pmFilename, false, false);
+	}
+
+	/**
+	 * The standard compute() method of the VoiceImportComponent interface.
+	 */
+	public boolean compute() throws IOException {
+		percent = 0;
+
+		String[] baseNameArray = bnl.getListAsArray();
+		System.out.println("Computing pitchmarks for " + baseNameArray.length + " utterances.");
+
+		/* Ensure the existence of the target pitchmark directory */
+		File dir = new File(getProp(PRAATPMDIR));
+		if (!dir.exists()) {
+			System.out.println("Creating the directory [" + getProp(PRAATPMDIR) + "].");
+			dir.mkdir();
+		}
+
+		// script.praat is provided as template.
+		InputStream scriptResource = getClass().getResourceAsStream("script.praat");
+		FileWriter scriptWriter = new FileWriter(tmpScript);
+		IOUtils.copy(scriptResource, scriptWriter);
+		scriptWriter.close();
+
+		// ensure that basenames.lst is up to date:
+		String baseNameListFileName = db.getProp("db.basenameFile");
+		bnl.write(baseNameListFileName);
+
+		// run Praat process:
+		String praatCommand = StringUtils.join(new String[] { getProp(COMMAND), tmpScript, baseNameListFileName,
+				getProp(WAVEDIR), getProp(PRAATPMDIR), getProp(MINPITCH), getProp(MAXPITCH) }, " ");
+		System.out.println("Running Praat as: " + praatCommand);
+		if (MaryUtils.isWindows()) {
+			praatCommand = "cmd.exe /c " + praatCommand;
+		}
+		General.launchProc(praatCommand, getName(), "");
+
+		// convert to EST format
+		for (int i = 0; i < baseNameArray.length; i++) {
+			percent = 100 * i / baseNameArray.length;
+			estPitchmarks(baseNameArray[i]);
+		}
+
+		return true;
+	}
+
+	/**
+	 * Provide the progress of computation, in percent, or -1 if that feature is not implemented.
+	 * 
+	 * @return -1 if not implemented, or an integer between 0 and 100.
+	 */
+	public int getProgress() {
+		// hack to get progress from Praat through file I/O:
+		File percentFile = new File(db.getProp(db.TEMPDIR) + "percent");
+		if (percentFile.exists()) {
+			percent = -1;
+			try {
+				FileReader percentReader = new FileReader(percentFile);
+				String percentString = IOUtils.toString(percentReader);
+				percent = Integer.parseInt(percentString);
+				percentReader.close();
+			} catch (IOException e) {
+				// TODO Auto-generated catch block
+				e.printStackTrace();
+			}
+		}
+		return percent;
+	}
 }
-

marytts-builder/src/main/java/marytts/tools/voiceimport/SCostUnitFileWriter.java

@@ -33,149 +33,137 @@ import marytts.unitselection.data.SCostFileReader;
 import marytts.unitselection.data.UnitFileReader;
 import marytts.util.data.MaryHeader;
 
-public class SCostUnitFileWriter extends VoiceImportComponent
-{
-    protected File maryDir;
-    protected String unitFileName;
-    protected File unitlabelDir;
-    protected int samplingRate;
-    protected String pauseSymbol;
-    protected PhoneLabelFeatureAligner aligner;
-    protected String unitlabelExt = ".hplab";
-        protected DatabaseLayout db = null;
-    protected int percent = 0;
-    
-    public String LABELDIR = "SCostUnitFileWriter.sCostlabelDir";
-    public String UNITFILE = "SCostUnitFileWriter.halfphoneunitFile";
-    public String SCOSTFILE = "SCostUnitFileWriter.sCostFile";
-    
-    public String getName(){
-        return "SCostUnitFileWriter";
-    }
-    
-    @Override
-    protected void initialiseComp()
-    throws Exception
-    {
-        maryDir = new File(db.getProp(db.FILEDIR));
-        
-        samplingRate = Integer.parseInt(db.getProp(db.SAMPLINGRATE));
-        pauseSymbol = System.getProperty("pause.symbol", "pau");
-    
-        unitFileName = getProp(UNITFILE);
-        unitlabelDir = new File(getProp(LABELDIR));
-        if (!unitlabelDir.exists()){
-            System.out.print(LABELDIR+" "+getProp(LABELDIR)
-                    +" does not exist; ");
-            if (!unitlabelDir.mkdir()){
-                throw new Error("Could not create LABELDIR");
-            }
-            System.out.print("Created successfully.\n");
-        }  
-        aligner = new PhoneLabelFeatureAligner();
-        db.initialiseComponent(aligner);        
-    }
-    
-    public SortedMap getDefaultProps(DatabaseLayout db){
-        this.db = db;
-       if (props == null){
-           props = new TreeMap();
-           String rootDir = db.getProp(db.ROOTDIR);
-           props.put(LABELDIR, rootDir
-                   +"sCost"+File.separator+"scostlabels"
-                   +System.getProperty("file.separator"));
-           props.put(UNITFILE, db.getProp(db.FILEDIR)
-                   +"halfphoneUnits"+db.getProp(db.MARYEXT)); 
-           props.put(SCOSTFILE, db.getProp(db.FILEDIR)
-                   +"sCostUnitsFile"+db.getProp(db.MARYEXT));           
-           
-       }
-       return props;
-    }
-    
-    protected void setupHelp(){
-        props2Help = new TreeMap();
-        props2Help.put(LABELDIR, "directory containing the phone labels");
-        props2Help.put(UNITFILE, "file containing all half phone units");
-        props2Help.put(SCOSTFILE, "file containing scost of halfphone units. Will be created by this module");
-    }
-    
-    @Override
-    public boolean compute() throws IOException, MaryConfigurationException
-    {
-        if (!unitlabelDir.exists()){
-            System.out.print(LABELDIR+" "+getProp(LABELDIR)+" does not exist; ");
-            throw new Error("Could not create LABELDIR");
-        }  
-        
-        ArrayList<Double> sCostList = new ArrayList<Double>();
-        for (int i=0; i<bnl.getLength(); i++) {
-            System.out.println("reading..."+ bnl.getName(i));
-            percent = 100*i/bnl.getLength();
-            String labFile = unitlabelDir +File.separator+ bnl.getName(i) + unitlabelExt;
-            UnitLabel[] uttData = UnitLabel.readLabFile(labFile);
-            sCostList.add(0.0);
-            for(int j=0; j<uttData.length; j++){
-                sCostList.add(uttData[j].getSCost());
-            }
-            sCostList.add(0.0);
-        }
-        int numberOfUnits = sCostList.size();
-        
-        UnitFileReader units = new UnitFileReader(getProp(UNITFILE));
-        //int noOfUnits = units.getNumberOfUnits();
-        if(numberOfUnits == units.getNumberOfUnits()){
-            System.out.println("Number of units in SCost file: "+numberOfUnits+" and no. of units in units file: "+units.getNumberOfUnits());
-        }
-        else{
-            System.out.println("ERROR: Number of units in SCost file: "+numberOfUnits+" and no. of units in units file: "+units.getNumberOfUnits());
-            return false;
-        }
-        System.out.println(numberOfUnits+" --> "+units.getNumberOfUnits());
-       
-        /**
-         * TODO : change file path
-         * 
-         */
-        DataOutputStream out = new DataOutputStream(new BufferedOutputStream(new FileOutputStream(getProp(SCOSTFILE))));
-        long posNumUnits = new MaryHeader(MaryHeader.SCOST).writeTo(out);
-        out.writeInt(numberOfUnits);
-        
-        for(int i=0; i<numberOfUnits; i++){
-            double sCost = sCostList.get(i).doubleValue();
-            out.writeFloat((float)sCost);
-        }
-        
-        out.close();
-        
-        SCostFileReader tester = new SCostFileReader(getProp(SCOSTFILE));
-        int unitsOnDisk = tester.getNumberOfUnits();
-        if (unitsOnDisk == numberOfUnits) {
-            System.out.println("Can read right number of units");
-            return true;
-        } else {
-            System.out.println("Read wrong number of units: "+unitsOnDisk);
-            return false;
-        }
-    }
-    
+public class SCostUnitFileWriter extends VoiceImportComponent {
+	protected File maryDir;
+	protected String unitFileName;
+	protected File unitlabelDir;
+	protected int samplingRate;
+	protected String pauseSymbol;
+	protected PhoneLabelFeatureAligner aligner;
+	protected String unitlabelExt = ".hplab";
+	protected DatabaseLayout db = null;
+	protected int percent = 0;
 
-    /**
-     * Provide the progress of computation, in percent, or -1 if
-     * that feature is not implemented.
-     * @return -1 if not implemented, or an integer between 0 and 100.
-     */
-    public int getProgress()
-    {
-        return percent;
-    }
+	public String LABELDIR = "SCostUnitFileWriter.sCostlabelDir";
+	public String UNITFILE = "SCostUnitFileWriter.halfphoneunitFile";
+	public String SCOSTFILE = "SCostUnitFileWriter.sCostFile";
 
-    public static void main(String[] args) throws Exception
-    {
-        SCostUnitFileWriter ufw = new SCostUnitFileWriter();
-        new DatabaseLayout(ufw); 
-        ufw.compute();
-    }
+	public String getName() {
+		return "SCostUnitFileWriter";
+	}
+
+	@Override
+	protected void initialiseComp() throws Exception {
+		maryDir = new File(db.getProp(db.FILEDIR));
+
+		samplingRate = Integer.parseInt(db.getProp(db.SAMPLINGRATE));
+		pauseSymbol = System.getProperty("pause.symbol", "pau");
+
+		unitFileName = getProp(UNITFILE);
+		unitlabelDir = new File(getProp(LABELDIR));
+		if (!unitlabelDir.exists()) {
+			System.out.print(LABELDIR + " " + getProp(LABELDIR) + " does not exist; ");
+			if (!unitlabelDir.mkdir()) {
+				throw new Error("Could not create LABELDIR");
+			}
+			System.out.print("Created successfully.\n");
+		}
+		aligner = new PhoneLabelFeatureAligner();
+		db.initialiseComponent(aligner);
+	}
+
+	public SortedMap getDefaultProps(DatabaseLayout db) {
+		this.db = db;
+		if (props == null) {
+			props = new TreeMap();
+			String rootDir = db.getProp(db.ROOTDIR);
+			props.put(LABELDIR, rootDir + "sCost" + File.separator + "scostlabels" + System.getProperty("file.separator"));
+			props.put(UNITFILE, db.getProp(db.FILEDIR) + "halfphoneUnits" + db.getProp(db.MARYEXT));
+			props.put(SCOSTFILE, db.getProp(db.FILEDIR) + "sCostUnitsFile" + db.getProp(db.MARYEXT));
+
+		}
+		return props;
+	}
+
+	protected void setupHelp() {
+		props2Help = new TreeMap();
+		props2Help.put(LABELDIR, "directory containing the phone labels");
+		props2Help.put(UNITFILE, "file containing all half phone units");
+		props2Help.put(SCOSTFILE, "file containing scost of halfphone units. Will be created by this module");
+	}
+
+	@Override
+	public boolean compute() throws IOException, MaryConfigurationException {
+		if (!unitlabelDir.exists()) {
+			System.out.print(LABELDIR + " " + getProp(LABELDIR) + " does not exist; ");
+			throw new Error("Could not create LABELDIR");
+		}
+
+		ArrayList<Double> sCostList = new ArrayList<Double>();
+		for (int i = 0; i < bnl.getLength(); i++) {
+			System.out.println("reading..." + bnl.getName(i));
+			percent = 100 * i / bnl.getLength();
+			String labFile = unitlabelDir + File.separator + bnl.getName(i) + unitlabelExt;
+			UnitLabel[] uttData = UnitLabel.readLabFile(labFile);
+			sCostList.add(0.0);
+			for (int j = 0; j < uttData.length; j++) {
+				sCostList.add(uttData[j].getSCost());
+			}
+			sCostList.add(0.0);
+		}
+		int numberOfUnits = sCostList.size();
+
+		UnitFileReader units = new UnitFileReader(getProp(UNITFILE));
+		// int noOfUnits = units.getNumberOfUnits();
+		if (numberOfUnits == units.getNumberOfUnits()) {
+			System.out.println("Number of units in SCost file: " + numberOfUnits + " and no. of units in units file: "
+					+ units.getNumberOfUnits());
+		} else {
+			System.out.println("ERROR: Number of units in SCost file: " + numberOfUnits + " and no. of units in units file: "
+					+ units.getNumberOfUnits());
+			return false;
+		}
+		System.out.println(numberOfUnits + " --> " + units.getNumberOfUnits());
+
+		/**
+		 * TODO : change file path
+		 * 
+		 */
+		DataOutputStream out = new DataOutputStream(new BufferedOutputStream(new FileOutputStream(getProp(SCOSTFILE))));
+		long posNumUnits = new MaryHeader(MaryHeader.SCOST).writeTo(out);
+		out.writeInt(numberOfUnits);
+
+		for (int i = 0; i < numberOfUnits; i++) {
+			double sCost = sCostList.get(i).doubleValue();
+			out.writeFloat((float) sCost);
+		}
+
+		out.close();
+
+		SCostFileReader tester = new SCostFileReader(getProp(SCOSTFILE));
+		int unitsOnDisk = tester.getNumberOfUnits();
+		if (unitsOnDisk == numberOfUnits) {
+			System.out.println("Can read right number of units");
+			return true;
+		} else {
+			System.out.println("Read wrong number of units: " + unitsOnDisk);
+			return false;
+		}
+	}
+
+	/**
+	 * Provide the progress of computation, in percent, or -1 if that feature is not implemented.
+	 * 
+	 * @return -1 if not implemented, or an integer between 0 and 100.
+	 */
+	public int getProgress() {
+		return percent;
+	}
+
+	public static void main(String[] args) throws Exception {
+		SCostUnitFileWriter ufw = new SCostUnitFileWriter();
+		new DatabaseLayout(ufw);
+		ufw.compute();
+	}
 
 }
-

marytts-builder/src/main/java/marytts/tools/voiceimport/SPTKMFCCExtractor.java

@@ -32,194 +32,189 @@ import java.util.Vector;
 import marytts.signalproc.analysis.Mfccs;
 import marytts.util.io.General;
 
-
 /**
- * A component for extracting mfcc files from wave files 
+ * A component for extracting mfcc files from wave files
+ * 
  * @author Marcela, Sathish Pammi
  *
  */
 public class SPTKMFCCExtractor extends VoiceImportComponent {
 
-    public final String SPTKINST = "SPTKMFCCExtractor.sptkinstallationDir";
-    public final String SOXCOMMAND = "SPTKMFCCExtractor.soxCommand";
-    public final String INWAVDIR = "SPTKMFCCExtractor.inputWAVEDIR";
-    public final String OUTMFCCDIR = "SPTKMFCCExtractor.outputMFCCDIR";
-    private DatabaseLayout db;    
-    private String sCostDirectory;
-    private String mfccExt = ".mfcc";
-    protected int percent = 0;
-    
-    protected void setupHelp() {
-        props2Help = new TreeMap();
-        props2Help.put(SOXCOMMAND,"Sox binary file absolute path.  ex: /usr/bin/sox ");
-        props2Help.put(SPTKINST,"SPTK installation directory. ex: /home/user/sw/SPTK-3.1/"); 
-        props2Help.put(INWAVDIR, "Input wave directory");
-        props2Help.put(OUTMFCCDIR, "Output MFCC directory");
-    }
-    
-    public SortedMap<String, String> getDefaultProps(DatabaseLayout db) {
-        this.db = db;
-        if (props == null){
-            props = new TreeMap();
-            props.put(SOXCOMMAND,"/usr/bin/sox");
-            props.put(SPTKINST,"/home/user/sw/SPTK-3.1/");
-            props.put(INWAVDIR, db.getProp(db.ROOTDIR)+File.separator+db.getProp(db.WAVDIR));
-            props.put(OUTMFCCDIR, db.getProp(db.ROOTDIR)+File.separator+"sCost"+File.separator+"wavemfcc");
-        }
-        return props;
-    }
+	public final String SPTKINST = "SPTKMFCCExtractor.sptkinstallationDir";
+	public final String SOXCOMMAND = "SPTKMFCCExtractor.soxCommand";
+	public final String INWAVDIR = "SPTKMFCCExtractor.inputWAVEDIR";
+	public final String OUTMFCCDIR = "SPTKMFCCExtractor.outputMFCCDIR";
+	private DatabaseLayout db;
+	private String sCostDirectory;
+	private String mfccExt = ".mfcc";
+	protected int percent = 0;
 
-    public final String getName() {
-        return "SPTKMFCCExtractor";
-    }
-    
-    @Override
-    protected void initialiseComp(){
-        // sCost dir creation, if doesn't exists
-        sCostDirectory = db.getProp(db.ROOTDIR)+File.separator+"sCost";
-        File sCostDir =  new File(sCostDirectory);
-        if(!sCostDir.exists()){
-            System.out.print(sCostDir.getAbsolutePath()+" does not exist; ");
-            if (!sCostDir.mkdir()){
-                throw new Error("Could not create "+sCostDir.getAbsolutePath());
-            }
-            System.out.print("Created successfully.\n");
-        }
-    }
+	protected void setupHelp() {
+		props2Help = new TreeMap();
+		props2Help.put(SOXCOMMAND, "Sox binary file absolute path.  ex: /usr/bin/sox ");
+		props2Help.put(SPTKINST, "SPTK installation directory. ex: /home/user/sw/SPTK-3.1/");
+		props2Help.put(INWAVDIR, "Input wave directory");
+		props2Help.put(OUTMFCCDIR, "Output MFCC directory");
+	}
 
-    public boolean compute() throws Exception {
-        
-        // check for SOX
-        File soxFile = new File(getProp(SOXCOMMAND));
-        if (!soxFile.exists()) {
-            throw new IOException("SOX command setting is wrong. Because file "+soxFile.getAbsolutePath()+" does not exist");
-        }
-        
-        // check for SPTK
-        File sptkFile = new File(getProp(SPTKINST)+File.separator+"bin"+File.separator+"mcep");
-        if (!sptkFile.exists()) {
-            throw new IOException("SPTK path setting is wrong. Because file "+sptkFile.getAbsolutePath()+" does not exist");
-        }
-        
-        // output mfcc dir creator 
-        File waveMFCCDir =  new File(getProp(OUTMFCCDIR));
-        if(!waveMFCCDir.exists()){
-            System.out.print(waveMFCCDir.getAbsolutePath()+" does not exist; ");
-            if (!waveMFCCDir.mkdir()){
-                throw new Error("Could not create "+waveMFCCDir.getAbsolutePath());
-            }
-            System.out.print("Created successfully.\n");
-        }
-        
-        // Now process all files, one by one
-        for (int i=0; i<bnl.getLength(); i++) {
-            percent = 100*i/bnl.getLength();
-            String baseName = bnl.getName(i);
-            String inFile = getProp(INWAVDIR)+File.separator+baseName+db.getProp(db.WAVEXT);
-            String outFile = getProp(OUTMFCCDIR)+File.separator+baseName+mfccExt;
-            getSptkMfcc(inFile,outFile);
-        }
-        
-        return true;
-    }
+	public SortedMap<String, String> getDefaultProps(DatabaseLayout db) {
+		this.db = db;
+		if (props == null) {
+			props = new TreeMap();
+			props.put(SOXCOMMAND, "/usr/bin/sox");
+			props.put(SPTKINST, "/home/user/sw/SPTK-3.1/");
+			props.put(INWAVDIR, db.getProp(db.ROOTDIR) + File.separator + db.getProp(db.WAVDIR));
+			props.put(OUTMFCCDIR, db.getProp(db.ROOTDIR) + File.separator + "sCost" + File.separator + "wavemfcc");
+		}
+		return props;
+	}
 
+	public final String getName() {
+		return "SPTKMFCCExtractor";
+	}
 
-    
-    
-    public int getProgress() {
-        return percent;
-    }
+	@Override
+	protected void initialiseComp() {
+		// sCost dir creation, if doesn't exists
+		sCostDirectory = db.getProp(db.ROOTDIR) + File.separator + "sCost";
+		File sCostDir = new File(sCostDirectory);
+		if (!sCostDir.exists()) {
+			System.out.print(sCostDir.getAbsolutePath() + " does not exist; ");
+			if (!sCostDir.mkdir()) {
+				throw new Error("Could not create " + sCostDir.getAbsolutePath());
+			}
+			System.out.print("Created successfully.\n");
+		}
+	}
 
-    /***
-     * Calculate mfcc using SPTK, uses sox to convert wav-->raw
-     * @throws IOException
-     * @throws InterruptedException
-     * @throws Exception
-     */
-    public void getSptkMfcc(String inFile, String outFile) throws IOException, InterruptedException, Exception{
-       
-       String tmpFile = db.getProp(db.TEMPDIR)+File.separator+"tmp.mfc"; 
-       String tmpRawFile = db.getProp(db.TEMPDIR)+File.separator+"tmp.raw";
-       String cmd;
-        
-       // SPTK parameters
-       int fs = 16000;
-       int frameLength = 400;
-       int frameLengthOutput = 512;
-       int framePeriod = 80;
-       int mgcOrder = 24;
-       int mgcDimension = 25;
-       // Mary header parameters
-       double ws = (frameLength/fs);   // window size in seconds 
-       double ss = (framePeriod/fs);   // skip size in seconds
-       
-       // SOX and SPTK commands
-       String sox = getProp(SOXCOMMAND);
-       String x2x = " "+getProp(SPTKINST)+File.separator+"/bin/x2x";
-       String frame = " "+getProp(SPTKINST)+File.separator+"/bin/frame";
-       String window = " "+getProp(SPTKINST)+File.separator+"/bin/window";
-       String mcep = " "+getProp(SPTKINST)+File.separator+"/bin/mcep";
-       String swab = " "+getProp(SPTKINST)+File.separator+"/bin/swab";
-       
-      
-       // convert the wav file to raw file with sox
-       cmd = sox + " " + inFile + " " +  tmpRawFile;
-       General.launchProc( cmd, "sox", inFile);
-       
-       System.out.println("Extracting MGC coefficients from " + inFile);
-       
-       cmd = x2x + " +sf " + tmpRawFile + " | " +
-                    frame + " +f -l " + frameLength + " -p " + framePeriod + " | " +
-                    window + " -l " + frameLength + " -L " + frameLengthOutput + " -w 1 -n 1 | " +
-                    mcep + " -a 0.42 -m " + mgcOrder + "  -l " + frameLengthOutput + " | " +
-                    swab + " +f > " + tmpFile;
-       
-       System.out.println("cmd=" + cmd);
-       General.launchBatchProc( cmd, "getSptkMfcc", inFile );
-       
-       // Now get the data and add the Mary header
-       int numFrames;
-       DataInputStream mfcData=null;
-       Vector <Float> mfc = new Vector<Float>();
-      
-       mfcData = new DataInputStream (new BufferedInputStream(new FileInputStream(tmpFile)));
-       try {
-         while(true){
-           mfc.add(mfcData.readFloat());
-         }         
-       } catch (EOFException e) { }
-       mfcData.close();
+	public boolean compute() throws Exception {
 
-       numFrames = mfc.size();
-       int numVectors = numFrames/mgcDimension;
-       Mfccs mgc = new Mfccs(numVectors, mgcDimension);
-       
-       int k=0;
-       for(int i=0; i<numVectors; i++){
-         for(int j=0; j<mgcDimension; j++){
-           mgc.mfccs[i][j] = mfc.get(k);
-           k++;
-         }
-       }
-       // Mary header parameters
-       mgc.params.samplingRate = fs;         /* samplingRateInHz */
-       mgc.params.skipsize     = (float)ss;  /* skipSizeInSeconds */
-       mgc.params.winsize      = (float)ws;  /* windowSizeInSeconds */
-       
-       mgc.writeMfccFile(outFile);
+		// check for SOX
+		File soxFile = new File(getProp(SOXCOMMAND));
+		if (!soxFile.exists()) {
+			throw new IOException("SOX command setting is wrong. Because file " + soxFile.getAbsolutePath() + " does not exist");
+		}
+
+		// check for SPTK
+		File sptkFile = new File(getProp(SPTKINST) + File.separator + "bin" + File.separator + "mcep");
+		if (!sptkFile.exists()) {
+			throw new IOException("SPTK path setting is wrong. Because file " + sptkFile.getAbsolutePath() + " does not exist");
+		}
+
+		// output mfcc dir creator
+		File waveMFCCDir = new File(getProp(OUTMFCCDIR));
+		if (!waveMFCCDir.exists()) {
+			System.out.print(waveMFCCDir.getAbsolutePath() + " does not exist; ");
+			if (!waveMFCCDir.mkdir()) {
+				throw new Error("Could not create " + waveMFCCDir.getAbsolutePath());
+			}
+			System.out.print("Created successfully.\n");
+		}
+
+		// Now process all files, one by one
+		for (int i = 0; i < bnl.getLength(); i++) {
+			percent = 100 * i / bnl.getLength();
+			String baseName = bnl.getName(i);
+			String inFile = getProp(INWAVDIR) + File.separator + baseName + db.getProp(db.WAVEXT);
+			String outFile = getProp(OUTMFCCDIR) + File.separator + baseName + mfccExt;
+			getSptkMfcc(inFile, outFile);
+		}
+
+		return true;
+	}
+
+	public int getProgress() {
+		return percent;
+	}
+
+	/***
+	 * Calculate mfcc using SPTK, uses sox to convert wav-->raw
+	 * 
+	 * @throws IOException
+	 * @throws InterruptedException
+	 * @throws Exception
+	 */
+	public void getSptkMfcc(String inFile, String outFile) throws IOException, InterruptedException, Exception {
+
+		String tmpFile = db.getProp(db.TEMPDIR) + File.separator + "tmp.mfc";
+		String tmpRawFile = db.getProp(db.TEMPDIR) + File.separator + "tmp.raw";
+		String cmd;
+
+		// SPTK parameters
+		int fs = 16000;
+		int frameLength = 400;
+		int frameLengthOutput = 512;
+		int framePeriod = 80;
+		int mgcOrder = 24;
+		int mgcDimension = 25;
+		// Mary header parameters
+		double ws = (frameLength / fs); // window size in seconds
+		double ss = (framePeriod / fs); // skip size in seconds
+
+		// SOX and SPTK commands
+		String sox = getProp(SOXCOMMAND);
+		String x2x = " " + getProp(SPTKINST) + File.separator + "/bin/x2x";
+		String frame = " " + getProp(SPTKINST) + File.separator + "/bin/frame";
+		String window = " " + getProp(SPTKINST) + File.separator + "/bin/window";
+		String mcep = " " + getProp(SPTKINST) + File.separator + "/bin/mcep";
+		String swab = " " + getProp(SPTKINST) + File.separator + "/bin/swab";
+
+		// convert the wav file to raw file with sox
+		cmd = sox + " " + inFile + " " + tmpRawFile;
+		General.launchProc(cmd, "sox", inFile);
+
+		System.out.println("Extracting MGC coefficients from " + inFile);
+
+		cmd = x2x + " +sf " + tmpRawFile + " | " + frame + " +f -l " + frameLength + " -p " + framePeriod + " | " + window
+				+ " -l " + frameLength + " -L " + frameLengthOutput + " -w 1 -n 1 | " + mcep + " -a 0.42 -m " + mgcOrder
+				+ "  -l " + frameLengthOutput + " | " + swab + " +f > " + tmpFile;
+
+		System.out.println("cmd=" + cmd);
+		General.launchBatchProc(cmd, "getSptkMfcc", inFile);
+
+		// Now get the data and add the Mary header
+		int numFrames;
+		DataInputStream mfcData = null;
+		Vector<Float> mfc = new Vector<Float>();
+
+		mfcData = new DataInputStream(new BufferedInputStream(new FileInputStream(tmpFile)));
+		try {
+			while (true) {
+				mfc.add(mfcData.readFloat());
+			}
+		} catch (EOFException e) {
+		}
+		mfcData.close();
+
+		numFrames = mfc.size();
+		int numVectors = numFrames / mgcDimension;
+		Mfccs mgc = new Mfccs(numVectors, mgcDimension);
+
+		int k = 0;
+		for (int i = 0; i < numVectors; i++) {
+			for (int j = 0; j < mgcDimension; j++) {
+				mgc.mfccs[i][j] = mfc.get(k);
+				k++;
+			}
+		}
+		// Mary header parameters
+		mgc.params.samplingRate = fs; /* samplingRateInHz */
+		mgc.params.skipsize = (float) ss; /* skipSizeInSeconds */
+		mgc.params.winsize = (float) ws; /* windowSizeInSeconds */
+
+		mgc.writeMfccFile(outFile);
+
+		// remove temp files
+		// 1. tempmfcc file
+		File tempFile = new File(tmpFile);
+		if (tempFile.exists()) {
+			tempFile.delete();
+		}
+		// 2. temp raw file
+		tempFile = new File(tmpRawFile);
+		if (tempFile.exists()) {
+			tempFile.delete();
+		}
+	}
 
-       // remove temp files
-       // 1. tempmfcc file
-       File tempFile = new File(tmpFile);
-       if(tempFile.exists()){
-           tempFile.delete();
-       }
-       // 2. temp raw file
-       tempFile = new File(tmpRawFile);
-       if(tempFile.exists()){
-           tempFile.delete();
-       }
-    }
-    
 }
-

marytts-builder/src/main/java/marytts/tools/voiceimport/SanityChecker.java

@@ -41,331 +41,318 @@ import marytts.features.FeatureDefinition;
 import marytts.features.FeatureVector;
 import marytts.unitselection.data.FeatureFileReader;
 
-class DiphoneCount implements Comparable<DiphoneCount>
-{
-    String diphone;
-    int count;
-    
-    DiphoneCount(String diphone, int count) {
-        this.diphone = diphone;
-        this.count = count;
-    }
-    
-    public int compareTo(DiphoneCount other) {
-        if (count == other.count) return 0;
-        if (count < other.count) return -1;
-        return 1;
-    }
-    
-    public boolean equals(Object dc)
-    {
-        if (!(dc instanceof DiphoneCount)) return false;
-        DiphoneCount other = (DiphoneCount) dc;
-        if (count == other.count) return true;
-        return false;
-    }
+class DiphoneCount implements Comparable<DiphoneCount> {
+	String diphone;
+	int count;
+
+	DiphoneCount(String diphone, int count) {
+		this.diphone = diphone;
+		this.count = count;
+	}
+
+	public int compareTo(DiphoneCount other) {
+		if (count == other.count)
+			return 0;
+		if (count < other.count)
+			return -1;
+		return 1;
+	}
+
+	public boolean equals(Object dc) {
+		if (!(dc instanceof DiphoneCount))
+			return false;
+		DiphoneCount other = (DiphoneCount) dc;
+		if (count == other.count)
+			return true;
+		return false;
+	}
 }
 
-
 /**
- * Sanity checker for voicebuilding 
- * One test case : check no. of diphones in database and cart tree
+ * Sanity checker for voicebuilding One test case : check no. of diphones in database and cart tree
+ * 
  * @author sathish pammi
  *
  */
 public class SanityChecker extends VoiceImportComponent {
 
-    Map<String, Set<Integer>> diPhoneSet;
-    Map<String, Set<Integer>> cartdiPhoneSet;
-    
-    public final String HALFFEATURES = "SanityChecker.halfPhoneFeatureFile";
-    public final String CARTTREE = "SanityChecker.cartTreeFile";
-    private DatabaseLayout db;    
-    
+	Map<String, Set<Integer>> diPhoneSet;
+	Map<String, Set<Integer>> cartdiPhoneSet;
 
-    protected void setupHelp() {
-        // TODO Auto-generated method stub
-        props2Help = new TreeMap();
-        props2Help.put(HALFFEATURES,"Half-phone feature file");
-        props2Help.put(CARTTREE,"CART tree file (file generated by CARTBuilder)"); 
-    }
-    
-    public SortedMap<String, String> getDefaultProps(DatabaseLayout db) {
-        this.db = db;
-        if (props == null){
-            props = new TreeMap();
-            
-            props.put(HALFFEATURES,db.getProp(db.ROOTDIR)+File.separator
-                    +"mary"+File.separator+"halfphoneFeatures.mry");
-            props.put(CARTTREE,db.getProp(db.ROOTDIR)+File.separator
-                         +"mary"+File.separator
-                         +"cart.mry");
-        }
-        return props;
-    }
+	public final String HALFFEATURES = "SanityChecker.halfPhoneFeatureFile";
+	public final String CARTTREE = "SanityChecker.cartTreeFile";
+	private DatabaseLayout db;
 
-    public final String getName() {
-        return "SanityChecker";
-    }
+	protected void setupHelp() {
+		// TODO Auto-generated method stub
+		props2Help = new TreeMap();
+		props2Help.put(HALFFEATURES, "Half-phone feature file");
+		props2Help.put(CARTTREE, "CART tree file (file generated by CARTBuilder)");
+	}
 
+	public SortedMap<String, String> getDefaultProps(DatabaseLayout db) {
+		this.db = db;
+		if (props == null) {
+			props = new TreeMap();
 
-    public boolean compute() throws Exception {
-        
-        // One test case : check no. of diphones in database and cart tree 
-        computeCoverage(getProp(HALFFEATURES));
-        computeCARTCoverage(getProp(CARTTREE), getProp(HALFFEATURES));
-        boolean success = printSanityCheckDetails(diPhoneSet, cartdiPhoneSet);
-        
-        return success;
-    }
+			props.put(HALFFEATURES, db.getProp(db.ROOTDIR) + File.separator + "mary" + File.separator + "halfphoneFeatures.mry");
+			props.put(CARTTREE, db.getProp(db.ROOTDIR) + File.separator + "mary" + File.separator + "cart.mry");
+		}
+		return props;
+	}
 
+	public final String getName() {
+		return "SanityChecker";
+	}
 
-    
-    
-    public int getProgress() {
-        return -1;
-    }
+	public boolean compute() throws Exception {
 
-    
-   public void computeCoverage(String inFile) throws IOException, MaryConfigurationException
-   {
-        FeatureFileReader ffr = FeatureFileReader.getFeatureFileReader(inFile);
-        FeatureVector[] fVCopy = ffr.getCopyOfFeatureVectors();
-        FeatureDefinition feaDef = ffr.getFeatureDefinition();
-        //int cUnitIdx; // current unit index
-        System.out.println("Features used to build voice : ");
-        System.out.println(feaDef.getFeatureNames());
-        int pIdx = feaDef.getFeatureIndex("phone");
-        int hpIdx = feaDef.getFeatureIndex("halfphone_unitname");
-        int lrIdx = feaDef.getFeatureIndex("halfphone_lr");
-        diPhoneSet = new HashMap<String, Set<Integer>>();
-        
-        for(int i=0; i < fVCopy.length; i++ ){
-            FeatureVector fv = fVCopy[i];
-            
-            int cUnitIdx = fv.getUnitIndex();
-            if(cUnitIdx <= 0) continue;
-            FeatureVector prevFV = fVCopy[i-1]; 
-            if(fv.getFeatureAsString(lrIdx, feaDef).equals("R")){
-                continue;
-            }
-            
-            String prevStr = prevFV.getFeatureAsString(hpIdx, feaDef);
-            String cStr = fv.getFeatureAsString(hpIdx, feaDef);
-            String prevPh =  prevFV.getFeatureAsString(pIdx, feaDef);
-            String cPh =  fv.getFeatureAsString(pIdx, feaDef);
-            String diPhone = prevPh +"_"+ cPh;
-            
-            if(diPhoneSet.containsKey(diPhone)){
-                Set<Integer> setList = (Set<Integer>) diPhoneSet.get(diPhone);
-                setList.add(cUnitIdx); 
-                diPhoneSet.put(diPhone, setList);
-                }
-                else {
-                    Set<Integer> setList = new HashSet<Integer>();
-                    setList.add(cUnitIdx);
-                    diPhoneSet.put(diPhone, setList);
-                }
-        }
-    }
-    
-    
-   private void computeCARTCoverage(String cartFile, String halfPhones) throws Exception{
-       
-       //ClassificationTree cart = new ClassificationTree();
-       CART cart = new CART();
-       MaryCARTReader mCart = new MaryCARTReader();
-       cart = mCart.load(cartFile);
-       
-       FeatureFileReader ffr = FeatureFileReader.getFeatureFileReader(halfPhones);
-       FeatureDefinition feaDef = ffr.getFeatureDefinition();
-       cartdiPhoneSet = new HashMap<String, Set<Integer>>();
-       int pIdx = feaDef.getFeatureIndex("phone");
-       int hpIdx = feaDef.getFeatureIndex("halfphone_unitname");
-       int lrIdx = feaDef.getFeatureIndex("halfphone_lr");
-       
-       ArrayList<Integer> listHalfUnits = new ArrayList<Integer>();
-       for (LeafNode leaf : cart.getLeafNodes()) {
-           int[] data = (int[])leaf.getAllData();
-           //System.out.println("Data SIZE : "+data.length);
-           for (int i=0; i<data.length; i++) {
-           //    System.out.println("Data Val: "+data[i]);
-               listHalfUnits.add(data[i]);
-           }
-       }
-       ArrayList<Integer> refList = new  ArrayList<Integer>();
-       refList = listHalfUnits;
-       //PrintWriter pw=new PrintWriter(new FileWriter("/home/sathish/Work/test/t2"));
-       
-       for ( Iterator<Integer> it = listHalfUnits.iterator(); it.hasNext(); ){
-           int unitIndex = ((Integer) it.next()).intValue();
-           if(unitIndex <= 0) continue;
-           int size = ffr.getNumberOfUnits();
-           if(unitIndex >= size){
-               System.err.println("Warning: Unit index "+unitIndex+" is greater that size "+size);
-               continue;
-           }
-           FeatureVector fv = ffr.getFeatureVector(unitIndex);
-           
-           if(fv.getFeatureAsString(lrIdx, feaDef).equals("R")){
-               continue;
-           }
-           
-           if(!containsUnitIndex(refList, unitIndex-1)){
-               continue;
-           }
-           
-           FeatureVector prevFV = ffr.getFeatureVector(unitIndex-1); 
-           String prevStr = prevFV.getFeatureAsString(hpIdx, feaDef);
-           String cStr = fv.getFeatureAsString(hpIdx, feaDef);
-           String prevPh =  prevFV.getFeatureAsString(pIdx, feaDef);
-           String cPh =  fv.getFeatureAsString(pIdx, feaDef);
-           String diPhone = prevPh +"_"+ cPh;
-           //System.out.println(unitIndex);
-           
-           if(cartdiPhoneSet.containsKey(diPhone)){
-               Set<Integer> setList = (Set<Integer>) cartdiPhoneSet.get(diPhone);
-               setList.add(unitIndex); 
-               cartdiPhoneSet.put(diPhone, setList);
-               }
-               else {
-                   Set<Integer> setList = new HashSet<Integer>();
-                   setList.add(unitIndex);
-                   cartdiPhoneSet.put(diPhone, setList);
-               }
-       }
-   }
-   
-    private boolean containsUnitIndex(ArrayList<Integer> refList, int uIndex){
-        
-        for ( Iterator<Integer> it = refList.iterator(); it.hasNext(); ){
-            int listVal = ((Integer) it.next()).intValue();
-            if(listVal == uIndex){
-                return true;
-            }
-        }
-        return false;
-    }
-    //<String, Set<Integer>>
-    private void printData(Map<String, Set<Integer>> mp, boolean printAll){
+		// One test case : check no. of diphones in database and cart tree
+		computeCoverage(getProp(HALFFEATURES));
+		computeCARTCoverage(getProp(CARTTREE), getProp(HALFFEATURES));
+		boolean success = printSanityCheckDetails(diPhoneSet, cartdiPhoneSet);
 
-        for ( Iterator it = mp.entrySet().iterator(); it.hasNext(); ) {
-            Map.Entry e = (Map.Entry) it.next();
-            Set<Integer> arr = (Set<Integer>) e.getValue();
-            String keyVal = (String) e.getKey();
-            int intVal = arr.size();
-            //System.out.println("Diphone: "+ keyVal+ " Count: "+intVal);
-            System.out.println(keyVal+" "+intVal);
-            if(printAll){
-                for ( Iterator sit = arr.iterator(); sit.hasNext(); ){
-                    System.out.print( "   "+ sit.next());
-                }
-                System.out.println();
-            }
-        }
-    }
-    
-    
- 
-    private void printDataToFile(Map mp, Map cartMap, String fileName) throws Exception{
-        
-        PrintWriter pw=new PrintWriter(new FileWriter(fileName));
-        
-        DiphoneCount[] mpSorted = new DiphoneCount[mp.entrySet().size()];
-        
-        int i=0;
-        for ( Iterator it = mp.entrySet().iterator(); it.hasNext(); ) {
-            Map.Entry e = (Map.Entry) it.next();
-            Set arr = (Set) e.getValue();
-            String keyVal = (String) e.getKey();
-            mpSorted[i] = new DiphoneCount(keyVal, arr.size());
-            i++;
-        }
-        Arrays.sort(mpSorted);
-        
-        for (i=0; i<mpSorted.length; i++) {
-            String diphone = mpSorted[i].diphone;
-            int mpCount = mpSorted[i].count;
-            int cartSetSize = 0;
-            if(cartMap.containsKey(diphone)){
-                Set cartSet = (Set)cartMap.get(diphone);
-                cartSetSize = cartSet.size();
-            }
-            System.out.println(diphone+" "+mpCount+" "+cartSetSize);
-            pw.println(diphone+" "+mpCount+" "+cartSetSize);
-        }
-        pw.flush();
-        pw.close();
-    }
-    
-    private boolean printSanityCheckDetails(Map mp, Map cartMap) throws Exception{
-        
-        DiphoneCount[] mpSorted = new DiphoneCount[mp.entrySet().size()];
-        
-        int i=0;
-        for ( Iterator it = mp.entrySet().iterator(); it.hasNext(); ) {
-            Map.Entry e = (Map.Entry) it.next();
-            Set arr = (Set) e.getValue();
-            String keyVal = (String) e.getKey();
-            mpSorted[i] = new DiphoneCount(keyVal, arr.size());
-            i++;
-        }
-        Arrays.sort(mpSorted);
-        
-        // First print all good things
-        for (i=0; i<mpSorted.length; i++) {
-            String diphone = mpSorted[i].diphone;
-            int mpCount = mpSorted[i].count;
-            int cartSetSize = 0;
-            if(cartMap.containsKey(diphone)){
-                Set cartSet = (Set)cartMap.get(diphone);
-                cartSetSize = cartSet.size();
-            }
-            if(mpCount == cartSetSize){
-                System.out.println("For diphone: "+diphone+" ; No. of diphones in database: "+mpCount+" ;  No. of diphones in CART tree:"+cartSetSize+ " --> OK. ");
-            }
-        }
-        
-        int failCount = 0;
-        // Next print all bad things
-        for (i=0; i<mpSorted.length; i++) {
-            String diphone = mpSorted[i].diphone;
-            int mpCount = mpSorted[i].count;
-            int cartSetSize = 0;
-            if(cartMap.containsKey(diphone)){
-                Set cartSet = (Set)cartMap.get(diphone);
-                cartSetSize = cartSet.size();
-            }
-            if(mpCount != cartSetSize){
-                System.out.println("WARNING :: For diphone: "+diphone+" ; No. of diphones in database: "+mpCount+" ;  No. of diphones in CART tree:"+cartSetSize+ " --> NOT OK. ");
-                failCount++;
-            }
-        }
-        
-        if(failCount > 0){
-            System.out.println("ERROR: Failed sanity check for "+failCount+" diphone units");
-            return false;
-        }
-        else return true;
-    }
-    
-    /**
-     * @param args
-     */
-    public static void main(String[] args) throws Exception {
+		return success;
+	}
 
-        String halfPhonesFile = "cmu-slt-arctic/mary/halfphoneFeatures.mry";
-        String cartFile = "cmu_us_slt_arctic/mary/cart.mry";
-        
-        SanityChecker dc = new SanityChecker();
-        dc.computeCoverage(halfPhonesFile);
-        dc.computeCARTCoverage(cartFile, halfPhonesFile);
-        dc.printSanityCheckDetails(dc.diPhoneSet, dc.cartdiPhoneSet);
-        
-    }
+	public int getProgress() {
+		return -1;
+	}
 
+	public void computeCoverage(String inFile) throws IOException, MaryConfigurationException {
+		FeatureFileReader ffr = FeatureFileReader.getFeatureFileReader(inFile);
+		FeatureVector[] fVCopy = ffr.getCopyOfFeatureVectors();
+		FeatureDefinition feaDef = ffr.getFeatureDefinition();
+		// int cUnitIdx; // current unit index
+		System.out.println("Features used to build voice : ");
+		System.out.println(feaDef.getFeatureNames());
+		int pIdx = feaDef.getFeatureIndex("phone");
+		int hpIdx = feaDef.getFeatureIndex("halfphone_unitname");
+		int lrIdx = feaDef.getFeatureIndex("halfphone_lr");
+		diPhoneSet = new HashMap<String, Set<Integer>>();
 
-    
+		for (int i = 0; i < fVCopy.length; i++) {
+			FeatureVector fv = fVCopy[i];
+
+			int cUnitIdx = fv.getUnitIndex();
+			if (cUnitIdx <= 0)
+				continue;
+			FeatureVector prevFV = fVCopy[i - 1];
+			if (fv.getFeatureAsString(lrIdx, feaDef).equals("R")) {
+				continue;
+			}
+
+			String prevStr = prevFV.getFeatureAsString(hpIdx, feaDef);
+			String cStr = fv.getFeatureAsString(hpIdx, feaDef);
+			String prevPh = prevFV.getFeatureAsString(pIdx, feaDef);
+			String cPh = fv.getFeatureAsString(pIdx, feaDef);
+			String diPhone = prevPh + "_" + cPh;
+
+			if (diPhoneSet.containsKey(diPhone)) {
+				Set<Integer> setList = (Set<Integer>) diPhoneSet.get(diPhone);
+				setList.add(cUnitIdx);
+				diPhoneSet.put(diPhone, setList);
+			} else {
+				Set<Integer> setList = new HashSet<Integer>();
+				setList.add(cUnitIdx);
+				diPhoneSet.put(diPhone, setList);
+			}
+		}
+	}
+
+	private void computeCARTCoverage(String cartFile, String halfPhones) throws Exception {
+
+		// ClassificationTree cart = new ClassificationTree();
+		CART cart = new CART();
+		MaryCARTReader mCart = new MaryCARTReader();
+		cart = mCart.load(cartFile);
+
+		FeatureFileReader ffr = FeatureFileReader.getFeatureFileReader(halfPhones);
+		FeatureDefinition feaDef = ffr.getFeatureDefinition();
+		cartdiPhoneSet = new HashMap<String, Set<Integer>>();
+		int pIdx = feaDef.getFeatureIndex("phone");
+		int hpIdx = feaDef.getFeatureIndex("halfphone_unitname");
+		int lrIdx = feaDef.getFeatureIndex("halfphone_lr");
+
+		ArrayList<Integer> listHalfUnits = new ArrayList<Integer>();
+		for (LeafNode leaf : cart.getLeafNodes()) {
+			int[] data = (int[]) leaf.getAllData();
+			// System.out.println("Data SIZE : "+data.length);
+			for (int i = 0; i < data.length; i++) {
+				// System.out.println("Data Val: "+data[i]);
+				listHalfUnits.add(data[i]);
+			}
+		}
+		ArrayList<Integer> refList = new ArrayList<Integer>();
+		refList = listHalfUnits;
+		// PrintWriter pw=new PrintWriter(new FileWriter("/home/sathish/Work/test/t2"));
+
+		for (Iterator<Integer> it = listHalfUnits.iterator(); it.hasNext();) {
+			int unitIndex = ((Integer) it.next()).intValue();
+			if (unitIndex <= 0)
+				continue;
+			int size = ffr.getNumberOfUnits();
+			if (unitIndex >= size) {
+				System.err.println("Warning: Unit index " + unitIndex + " is greater that size " + size);
+				continue;
+			}
+			FeatureVector fv = ffr.getFeatureVector(unitIndex);
+
+			if (fv.getFeatureAsString(lrIdx, feaDef).equals("R")) {
+				continue;
+			}
+
+			if (!containsUnitIndex(refList, unitIndex - 1)) {
+				continue;
+			}
+
+			FeatureVector prevFV = ffr.getFeatureVector(unitIndex - 1);
+			String prevStr = prevFV.getFeatureAsString(hpIdx, feaDef);
+			String cStr = fv.getFeatureAsString(hpIdx, feaDef);
+			String prevPh = prevFV.getFeatureAsString(pIdx, feaDef);
+			String cPh = fv.getFeatureAsString(pIdx, feaDef);
+			String diPhone = prevPh + "_" + cPh;
+			// System.out.println(unitIndex);
+
+			if (cartdiPhoneSet.containsKey(diPhone)) {
+				Set<Integer> setList = (Set<Integer>) cartdiPhoneSet.get(diPhone);
+				setList.add(unitIndex);
+				cartdiPhoneSet.put(diPhone, setList);
+			} else {
+				Set<Integer> setList = new HashSet<Integer>();
+				setList.add(unitIndex);
+				cartdiPhoneSet.put(diPhone, setList);
+			}
+		}
+	}
+
+	private boolean containsUnitIndex(ArrayList<Integer> refList, int uIndex) {
+
+		for (Iterator<Integer> it = refList.iterator(); it.hasNext();) {
+			int listVal = ((Integer) it.next()).intValue();
+			if (listVal == uIndex) {
+				return true;
+			}
+		}
+		return false;
+	}
+
+	// <String, Set<Integer>>
+	private void printData(Map<String, Set<Integer>> mp, boolean printAll) {
+
+		for (Iterator it = mp.entrySet().iterator(); it.hasNext();) {
+			Map.Entry e = (Map.Entry) it.next();
+			Set<Integer> arr = (Set<Integer>) e.getValue();
+			String keyVal = (String) e.getKey();
+			int intVal = arr.size();
+			// System.out.println("Diphone: "+ keyVal+ " Count: "+intVal);
+			System.out.println(keyVal + " " + intVal);
+			if (printAll) {
+				for (Iterator sit = arr.iterator(); sit.hasNext();) {
+					System.out.print("   " + sit.next());
+				}
+				System.out.println();
+			}
+		}
+	}
+
+	private void printDataToFile(Map mp, Map cartMap, String fileName) throws Exception {
+
+		PrintWriter pw = new PrintWriter(new FileWriter(fileName));
+
+		DiphoneCount[] mpSorted = new DiphoneCount[mp.entrySet().size()];
+
+		int i = 0;
+		for (Iterator it = mp.entrySet().iterator(); it.hasNext();) {
+			Map.Entry e = (Map.Entry) it.next();
+			Set arr = (Set) e.getValue();
+			String keyVal = (String) e.getKey();
+			mpSorted[i] = new DiphoneCount(keyVal, arr.size());
+			i++;
+		}
+		Arrays.sort(mpSorted);
+
+		for (i = 0; i < mpSorted.length; i++) {
+			String diphone = mpSorted[i].diphone;
+			int mpCount = mpSorted[i].count;
+			int cartSetSize = 0;
+			if (cartMap.containsKey(diphone)) {
+				Set cartSet = (Set) cartMap.get(diphone);
+				cartSetSize = cartSet.size();
+			}
+			System.out.println(diphone + " " + mpCount + " " + cartSetSize);
+			pw.println(diphone + " " + mpCount + " " + cartSetSize);
+		}
+		pw.flush();
+		pw.close();
+	}
+
+	private boolean printSanityCheckDetails(Map mp, Map cartMap) throws Exception {
+
+		DiphoneCount[] mpSorted = new DiphoneCount[mp.entrySet().size()];
+
+		int i = 0;
+		for (Iterator it = mp.entrySet().iterator(); it.hasNext();) {
+			Map.Entry e = (Map.Entry) it.next();
+			Set arr = (Set) e.getValue();
+			String keyVal = (String) e.getKey();
+			mpSorted[i] = new DiphoneCount(keyVal, arr.size());
+			i++;
+		}
+		Arrays.sort(mpSorted);
+
+		// First print all good things
+		for (i = 0; i < mpSorted.length; i++) {
+			String diphone = mpSorted[i].diphone;
+			int mpCount = mpSorted[i].count;
+			int cartSetSize = 0;
+			if (cartMap.containsKey(diphone)) {
+				Set cartSet = (Set) cartMap.get(diphone);
+				cartSetSize = cartSet.size();
+			}
+			if (mpCount == cartSetSize) {
+				System.out.println("For diphone: " + diphone + " ; No. of diphones in database: " + mpCount
+						+ " ;  No. of diphones in CART tree:" + cartSetSize + " --> OK. ");
+			}
+		}
+
+		int failCount = 0;
+		// Next print all bad things
+		for (i = 0; i < mpSorted.length; i++) {
+			String diphone = mpSorted[i].diphone;
+			int mpCount = mpSorted[i].count;
+			int cartSetSize = 0;
+			if (cartMap.containsKey(diphone)) {
+				Set cartSet = (Set) cartMap.get(diphone);
+				cartSetSize = cartSet.size();
+			}
+			if (mpCount != cartSetSize) {
+				System.out.println("WARNING :: For diphone: " + diphone + " ; No. of diphones in database: " + mpCount
+						+ " ;  No. of diphones in CART tree:" + cartSetSize + " --> NOT OK. ");
+				failCount++;
+			}
+		}
+
+		if (failCount > 0) {
+			System.out.println("ERROR: Failed sanity check for " + failCount + " diphone units");
+			return false;
+		} else
+			return true;
+	}
+
+	/**
+	 * @param args
+	 */
+	public static void main(String[] args) throws Exception {
+
+		String halfPhonesFile = "cmu-slt-arctic/mary/halfphoneFeatures.mry";
+		String cartFile = "cmu_us_slt_arctic/mary/cart.mry";
+
+		SanityChecker dc = new SanityChecker();
+		dc.computeCoverage(halfPhonesFile);
+		dc.computeCARTCoverage(cartFile, halfPhonesFile);
+		dc.printSanityCheckDetails(dc.diPhoneSet, dc.cartdiPhoneSet);
+
+	}
 
 }
-

marytts-builder/src/main/java/marytts/tools/voiceimport/SettingsGUI.java

@@ -49,329 +49,318 @@ import javax.swing.event.TableModelEvent;
 import javax.swing.table.AbstractTableModel;
 import javax.swing.table.TableCellEditor;
 
-
 public class SettingsGUI {
-   
-    private final JFrame frame = new JFrame("Settings Editor");
-    private DatabaseLayout db;
-    private String[][] tableProps;
-    private PropTableModel tableModel;
-    private JTable table;
-    private String[] compNames;
-    private JScrollPane scrollPane;
-    private JComboBox componentsComboBox;
-    private final Map<String,String> comps2HelpText;
-    private String simpleModeHelpText;
-    private String guiText; 
-    private boolean wasSaved;
-    
-    public SettingsGUI(DatabaseLayout db, SortedMap<String,String> props, String simpleModeHelpText, String guiText)
-    {
-        this.db = db;
-        this.simpleModeHelpText = simpleModeHelpText;
-        this.guiText = guiText;
-        comps2HelpText = null;
-         Set<String> propSet = props.keySet();
-         List<String[]> propList = new ArrayList<String[]>();
-        for (String key : props.keySet()) {
-            Object value = props.get(key);
-            if (value instanceof String) {
-                //this is a global prop
-                if (db.isEditable((String)value)){
-                    String[] keyAndValue = new String[2];
-                    keyAndValue[0] = key;
-                    keyAndValue[1] = (String) value;
-                    propList.add(keyAndValue);
-                }
-            } else {
-                //these are props for a component
-                if (value instanceof SortedMap){
-                    SortedMap<String,String> newLocalProps = new TreeMap<String, String>();
-                    SortedMap<String, String> mapValue = (SortedMap) value;
-                    for (String nextKey : mapValue.keySet()) {
-                        String nextValue = mapValue.get(nextKey);
-                        String[] keyAndValue = new String[2];
-                        keyAndValue[0] = nextKey;
-                        keyAndValue[1] = nextValue;
-                        propList.add(keyAndValue);
-                    }
-                }
-            }
-        }//end of loop over props
-        tableProps = new String[propList.size()][];
-        for (int i=0;i<propList.size();i++){
-            tableProps[i] = (String[]) propList.get(i);            
-        }
-        
-        display(null, true);
-    }
-    
-    public SettingsGUI(DatabaseLayout db, 
-            			String[][] props,
-            			String selectedComp,
-            			Map<String,String> comps2HelpText)
-    {
-        this.db = db;        
-        this.tableProps = props;
-        this.comps2HelpText = comps2HelpText;
-        display(selectedComp,false);
-    }
-    
-    /**
-     * Show a frame displaying the help file.
-     * @param db the DatbaseLayout
-     * @param props the properties and values to be displayed
-     * @return true, if no error occurred
-     */
-    public void display(String selectedComp, boolean simpleMode)
-    {
-        wasSaved = false;
-        //final JFrame frame = new JFrame("Settings Editor");
-        GridBagLayout gridBagLayout = new GridBagLayout();
-        GridBagConstraints gridC = new GridBagConstraints();
-        frame.getContentPane().setLayout( gridBagLayout );
-        if (simpleMode) {
-            JLabel guiTextLabel = new JLabel(guiText);
-            gridC.gridx = 0;
-            gridC.gridy = 0;
-            gridBagLayout.setConstraints(guiTextLabel, gridC );
-            frame.getContentPane().add(guiTextLabel);
-            String[] columnNames = {"Property", "Value"};
-            tableModel = new PropTableModel(columnNames,tableProps);
-        } else {
-            compNames = db.getCompNamesForDisplay();        
-            componentsComboBox = new JComboBox(compNames);
-            componentsComboBox.setSelectedItem(selectedComp);
-            componentsComboBox.addActionListener(new ActionListener() {
-                public void actionPerformed(ActionEvent e) {
-                    JComboBox cb = (JComboBox)e.getSource();
-                    String compName = (String)cb.getSelectedItem();
-                    updateTable(compName);
-                }
-            });
-            gridC.gridx = 0;
-            gridC.gridy = 0;
-            gridBagLayout.setConstraints(componentsComboBox , gridC );
-            frame.getContentPane().add(componentsComboBox);
-            
-            //build a new JTable        
-            String[] columnNames = {"Property", "Value"};
-            String[][] currentProps = getPropsForCompName(selectedComp);
-            tableModel = new PropTableModel(columnNames,currentProps);
-        }
-        table = new JTable(tableModel); 
-        //set the focus traversal keys for the table
-        Set<KeyStroke> forwardKeys = new HashSet<KeyStroke>();
-        forwardKeys.add(KeyStroke.getKeyStroke(KeyEvent.VK_TAB, 0, false));
-        table.setFocusTraversalKeys(KeyboardFocusManager.FORWARD_TRAVERSAL_KEYS,forwardKeys);
-        Set<KeyStroke> backwardKeys = new HashSet<KeyStroke>();
-        backwardKeys.add(KeyStroke.getKeyStroke(KeyEvent.VK_TAB,KeyEvent.SHIFT_MASK+KeyEvent.SHIFT_DOWN_MASK, false));
-        table.setFocusTraversalKeys(KeyboardFocusManager.BACKWARD_TRAVERSAL_KEYS,backwardKeys);
-        //table.setPreferredScrollableViewportSize(new Dimension(600, 500));
-        scrollPane = new JScrollPane(table);
-        gridC.gridy = 1;
-        // resize scroll pane:
-        gridC.weightx = 1;
-        gridC.weighty = 1;
-        gridC.fill = GridBagConstraints.HORIZONTAL;        
-        scrollPane.setPreferredSize(new Dimension(600, 300));
-        gridBagLayout.setConstraints( scrollPane, gridC );
-        
-        frame.getContentPane().add(scrollPane);
-        gridC.gridy = 2;
-        // do not resize buttons:
-        gridC.weightx = 0;
-        gridC.weighty = 0;
-        JButton helpButton = new JButton("Help");
-        helpButton.setMnemonic(KeyEvent.VK_H);
-        helpButton.addActionListener(new ActionListener() {
-            public void actionPerformed(ActionEvent e) {
-                    new Thread("DisplayHelpGUIThread") {
-                        public void run() {    
-                            if (componentsComboBox == null){
-                                new HelpGUI(simpleModeHelpText).display();  
-                            } else {
-                                String helpText = 
-                                    (String) comps2HelpText.get(componentsComboBox.getSelectedItem());
-                                new HelpGUI(helpText).display();      
-                            }
-                        }}.start();
-            }
-        });
-        JButton saveButton = new JButton("Save");
-        saveButton.setMnemonic(KeyEvent.VK_S);
-        saveButton.addActionListener(new ActionListener() {
-            public void actionPerformed(ActionEvent e) {
-                stopTableEditing();
-                updateProps();
-                wasSaved = true;
-                frame.setVisible(false);
-            }
-        });
-        JButton cancelButton = new JButton("Cancel");
-        cancelButton.setMnemonic(KeyEvent.VK_C);
-        cancelButton.addActionListener(new ActionListener() {
-            public void actionPerformed(ActionEvent e) {
-                wasSaved = false;
-                frame.setVisible(false);
-            }
-        });
-        JPanel buttonPanel = new JPanel();
-        buttonPanel.setLayout(new FlowLayout());
-        buttonPanel.add(helpButton);
-        buttonPanel.add(saveButton);
-        buttonPanel.add(cancelButton);
-        gridBagLayout.setConstraints( buttonPanel, gridC );
-        frame.getContentPane().add(buttonPanel);
-        frame.addWindowListener(new WindowAdapter() {
-            public void windowClosing(WindowEvent evt) {
-                wasSaved = false;
-                frame.setVisible(false);
-            }
-        });
-        frame.pack();
-        frame.setVisible(true);
-        do {
-            try {
-                Thread.sleep(10); // OK, this is ugly, but I don't mind today...
-            } catch (InterruptedException e) {
-            }
-        } while (frame.isVisible());
-        
-        frame.dispose(); 
-    }
-    
-    public boolean wasSaved(){
-        return wasSaved;
-    }
-    
-    private String[][] getPropsForCompName(String name){
-        if (name.equals("Global properties"))
-            name = "db";
-        List<String[]> propList = new ArrayList<String[]>();
-        for (int i=0;i<tableProps.length;i++){
-            String[] keyAndValue = tableProps[i];
-            //System.err.println(keyAndValue[0]+" --- "+name);
-            if (keyAndValue[0].startsWith(name+".")){
-                propList.add(keyAndValue);
-            }
-        }
-        String[][] result = new String[propList.size()][];
-        for (int i=0;i<propList.size();i++){
-            result[i] = propList.get(i);
-        }
-        return result;        
-    }
-    
-    private void updateProps() {           
-        db.updateProps(tableProps);
-    }
-    
-    private void stopTableEditing() {
-        if (table.isEditing()) {
-            TableCellEditor ed = table.getCellEditor();
-            assert ed != null;
-            boolean success = ed.stopCellEditing(); // we first try to save
-            if (!success) {
-                ed.cancelCellEditing();
-            }
-            assert !table.isEditing();
-        }
-    }
-    
-    private void updateTable(String compName){
-        // First, make sure that any field that is currently being edited is saved or discarded:
-        stopTableEditing();
-        // Then, update the table model:
-        String[][] currentProps = getPropsForCompName(compName);
-        tableModel.setProps(currentProps);
-        table.tableChanged(new TableModelEvent(tableModel));
-        //int hsize = currentProps.length*20;
-        //scrollPane.setPreferredSize(new Dimension(600,hsize));
-        //frame.pack();
-    }
-    
-    private class PropTableModel extends AbstractTableModel
-    {
-        private String[] columnNames;
-        private boolean DEBUG = false;
-        private String[][] props;
 
-        public PropTableModel(String[] columnNames,
-                			String[][] props){
-            this.columnNames = columnNames;
-            this.props = props;
-        }
-        
-        public void setProps(String[][] props){
-            this.props = props;
-        }
-        
-        public int getColumnCount() {
-            return columnNames.length;
-        }
+	private final JFrame frame = new JFrame("Settings Editor");
+	private DatabaseLayout db;
+	private String[][] tableProps;
+	private PropTableModel tableModel;
+	private JTable table;
+	private String[] compNames;
+	private JScrollPane scrollPane;
+	private JComboBox componentsComboBox;
+	private final Map<String, String> comps2HelpText;
+	private String simpleModeHelpText;
+	private String guiText;
+	private boolean wasSaved;
 
-        public int getRowCount() {
-            return props.length;
-        }
+	public SettingsGUI(DatabaseLayout db, SortedMap<String, String> props, String simpleModeHelpText, String guiText) {
+		this.db = db;
+		this.simpleModeHelpText = simpleModeHelpText;
+		this.guiText = guiText;
+		comps2HelpText = null;
+		Set<String> propSet = props.keySet();
+		List<String[]> propList = new ArrayList<String[]>();
+		for (String key : props.keySet()) {
+			Object value = props.get(key);
+			if (value instanceof String) {
+				// this is a global prop
+				if (db.isEditable((String) value)) {
+					String[] keyAndValue = new String[2];
+					keyAndValue[0] = key;
+					keyAndValue[1] = (String) value;
+					propList.add(keyAndValue);
+				}
+			} else {
+				// these are props for a component
+				if (value instanceof SortedMap) {
+					SortedMap<String, String> newLocalProps = new TreeMap<String, String>();
+					SortedMap<String, String> mapValue = (SortedMap) value;
+					for (String nextKey : mapValue.keySet()) {
+						String nextValue = mapValue.get(nextKey);
+						String[] keyAndValue = new String[2];
+						keyAndValue[0] = nextKey;
+						keyAndValue[1] = nextValue;
+						propList.add(keyAndValue);
+					}
+				}
+			}
+		}// end of loop over props
+		tableProps = new String[propList.size()][];
+		for (int i = 0; i < propList.size(); i++) {
+			tableProps[i] = (String[]) propList.get(i);
+		}
 
-        public String getColumnName(int col) {
-            return columnNames[col];
-        }
+		display(null, true);
+	}
 
-        public Object getValueAt(int row, int col) {
-            return props[row][col];
-        }
-        
-        /*
-         * JTable uses this method to determine the default renderer/
-         * editor for each cell.
-         */
-        public Class getColumnClass(int c) {
-            return String.class;
-        }
+	public SettingsGUI(DatabaseLayout db, String[][] props, String selectedComp, Map<String, String> comps2HelpText) {
+		this.db = db;
+		this.tableProps = props;
+		this.comps2HelpText = comps2HelpText;
+		display(selectedComp, false);
+	}
 
-        
-        public boolean isCellEditable(int row, int col) {
-            //Note that the data/cell address is constant,
-            //no matter where the cell appears onscreen.
-            return col != 0;
-        }
+	/**
+	 * Show a frame displaying the help file.
+	 * 
+	 * @param db
+	 *            the DatbaseLayout
+	 * @param props
+	 *            the properties and values to be displayed
+	 * @return true, if no error occurred
+	 */
+	public void display(String selectedComp, boolean simpleMode) {
+		wasSaved = false;
+		// final JFrame frame = new JFrame("Settings Editor");
+		GridBagLayout gridBagLayout = new GridBagLayout();
+		GridBagConstraints gridC = new GridBagConstraints();
+		frame.getContentPane().setLayout(gridBagLayout);
+		if (simpleMode) {
+			JLabel guiTextLabel = new JLabel(guiText);
+			gridC.gridx = 0;
+			gridC.gridy = 0;
+			gridBagLayout.setConstraints(guiTextLabel, gridC);
+			frame.getContentPane().add(guiTextLabel);
+			String[] columnNames = { "Property", "Value" };
+			tableModel = new PropTableModel(columnNames, tableProps);
+		} else {
+			compNames = db.getCompNamesForDisplay();
+			componentsComboBox = new JComboBox(compNames);
+			componentsComboBox.setSelectedItem(selectedComp);
+			componentsComboBox.addActionListener(new ActionListener() {
+				public void actionPerformed(ActionEvent e) {
+					JComboBox cb = (JComboBox) e.getSource();
+					String compName = (String) cb.getSelectedItem();
+					updateTable(compName);
+				}
+			});
+			gridC.gridx = 0;
+			gridC.gridy = 0;
+			gridBagLayout.setConstraints(componentsComboBox, gridC);
+			frame.getContentPane().add(componentsComboBox);
 
+			// build a new JTable
+			String[] columnNames = { "Property", "Value" };
+			String[][] currentProps = getPropsForCompName(selectedComp);
+			tableModel = new PropTableModel(columnNames, currentProps);
+		}
+		table = new JTable(tableModel);
+		// set the focus traversal keys for the table
+		Set<KeyStroke> forwardKeys = new HashSet<KeyStroke>();
+		forwardKeys.add(KeyStroke.getKeyStroke(KeyEvent.VK_TAB, 0, false));
+		table.setFocusTraversalKeys(KeyboardFocusManager.FORWARD_TRAVERSAL_KEYS, forwardKeys);
+		Set<KeyStroke> backwardKeys = new HashSet<KeyStroke>();
+		backwardKeys.add(KeyStroke.getKeyStroke(KeyEvent.VK_TAB, KeyEvent.SHIFT_MASK + KeyEvent.SHIFT_DOWN_MASK, false));
+		table.setFocusTraversalKeys(KeyboardFocusManager.BACKWARD_TRAVERSAL_KEYS, backwardKeys);
+		// table.setPreferredScrollableViewportSize(new Dimension(600, 500));
+		scrollPane = new JScrollPane(table);
+		gridC.gridy = 1;
+		// resize scroll pane:
+		gridC.weightx = 1;
+		gridC.weighty = 1;
+		gridC.fill = GridBagConstraints.HORIZONTAL;
+		scrollPane.setPreferredSize(new Dimension(600, 300));
+		gridBagLayout.setConstraints(scrollPane, gridC);
 
-        public void setValueAt(Object value, int row, int col) {
-            if (DEBUG) {
-                System.out.println("Setting value at " + row + "," + col
-                                   + " to " + value);
-            }
-            props[row][col] = (String)value;
-            fireTableCellUpdated(row, col);
-        }
+		frame.getContentPane().add(scrollPane);
+		gridC.gridy = 2;
+		// do not resize buttons:
+		gridC.weightx = 0;
+		gridC.weighty = 0;
+		JButton helpButton = new JButton("Help");
+		helpButton.setMnemonic(KeyEvent.VK_H);
+		helpButton.addActionListener(new ActionListener() {
+			public void actionPerformed(ActionEvent e) {
+				new Thread("DisplayHelpGUIThread") {
+					public void run() {
+						if (componentsComboBox == null) {
+							new HelpGUI(simpleModeHelpText).display();
+						} else {
+							String helpText = (String) comps2HelpText.get(componentsComboBox.getSelectedItem());
+							new HelpGUI(helpText).display();
+						}
+					}
+				}.start();
+			}
+		});
+		JButton saveButton = new JButton("Save");
+		saveButton.setMnemonic(KeyEvent.VK_S);
+		saveButton.addActionListener(new ActionListener() {
+			public void actionPerformed(ActionEvent e) {
+				stopTableEditing();
+				updateProps();
+				wasSaved = true;
+				frame.setVisible(false);
+			}
+		});
+		JButton cancelButton = new JButton("Cancel");
+		cancelButton.setMnemonic(KeyEvent.VK_C);
+		cancelButton.addActionListener(new ActionListener() {
+			public void actionPerformed(ActionEvent e) {
+				wasSaved = false;
+				frame.setVisible(false);
+			}
+		});
+		JPanel buttonPanel = new JPanel();
+		buttonPanel.setLayout(new FlowLayout());
+		buttonPanel.add(helpButton);
+		buttonPanel.add(saveButton);
+		buttonPanel.add(cancelButton);
+		gridBagLayout.setConstraints(buttonPanel, gridC);
+		frame.getContentPane().add(buttonPanel);
+		frame.addWindowListener(new WindowAdapter() {
+			public void windowClosing(WindowEvent evt) {
+				wasSaved = false;
+				frame.setVisible(false);
+			}
+		});
+		frame.pack();
+		frame.setVisible(true);
+		do {
+			try {
+				Thread.sleep(10); // OK, this is ugly, but I don't mind today...
+			} catch (InterruptedException e) {
+			}
+		} while (frame.isVisible());
 
-        private void printDebugData() {
-            int numRows = getRowCount();
-            int numCols = getColumnCount();
+		frame.dispose();
+	}
 
-            for (int i=0; i < numRows; i++) {
-                System.out.print("    row " + i + ":");
-                for (int j=0; j < numCols; j++) {
-                    System.out.print("  " + props[i][j]);
-                }
-                System.out.println();
-            }
-            System.out.println("--------------------------");
-        }
-    }
-    
-    class HelpButtonActionListener implements ActionListener
-    {
-        public void actionPerformed(ActionEvent e) {
-            new Thread("DisplayHelpGUIThread") {
-                public void run() {
+	public boolean wasSaved() {
+		return wasSaved;
+	}
+
+	private String[][] getPropsForCompName(String name) {
+		if (name.equals("Global properties"))
+			name = "db";
+		List<String[]> propList = new ArrayList<String[]>();
+		for (int i = 0; i < tableProps.length; i++) {
+			String[] keyAndValue = tableProps[i];
+			// System.err.println(keyAndValue[0]+" --- "+name);
+			if (keyAndValue[0].startsWith(name + ".")) {
+				propList.add(keyAndValue);
+			}
+		}
+		String[][] result = new String[propList.size()][];
+		for (int i = 0; i < propList.size(); i++) {
+			result[i] = propList.get(i);
+		}
+		return result;
+	}
+
+	private void updateProps() {
+		db.updateProps(tableProps);
+	}
+
+	private void stopTableEditing() {
+		if (table.isEditing()) {
+			TableCellEditor ed = table.getCellEditor();
+			assert ed != null;
+			boolean success = ed.stopCellEditing(); // we first try to save
+			if (!success) {
+				ed.cancelCellEditing();
+			}
+			assert !table.isEditing();
+		}
+	}
+
+	private void updateTable(String compName) {
+		// First, make sure that any field that is currently being edited is saved or discarded:
+		stopTableEditing();
+		// Then, update the table model:
+		String[][] currentProps = getPropsForCompName(compName);
+		tableModel.setProps(currentProps);
+		table.tableChanged(new TableModelEvent(tableModel));
+		// int hsize = currentProps.length*20;
+		// scrollPane.setPreferredSize(new Dimension(600,hsize));
+		// frame.pack();
+	}
+
+	private class PropTableModel extends AbstractTableModel {
+		private String[] columnNames;
+		private boolean DEBUG = false;
+		private String[][] props;
+
+		public PropTableModel(String[] columnNames, String[][] props) {
+			this.columnNames = columnNames;
+			this.props = props;
+		}
+
+		public void setProps(String[][] props) {
+			this.props = props;
+		}
+
+		public int getColumnCount() {
+			return columnNames.length;
+		}
+
+		public int getRowCount() {
+			return props.length;
+		}
+
+		public String getColumnName(int col) {
+			return columnNames[col];
+		}
+
+		public Object getValueAt(int row, int col) {
+			return props[row][col];
+		}
+
+		/*
+		 * JTable uses this method to determine the default renderer/ editor for each cell.
+		 */
+		public Class getColumnClass(int c) {
+			return String.class;
+		}
+
+		public boolean isCellEditable(int row, int col) {
+			// Note that the data/cell address is constant,
+			// no matter where the cell appears onscreen.
+			return col != 0;
+		}
+
+		public void setValueAt(Object value, int row, int col) {
+			if (DEBUG) {
+				System.out.println("Setting value at " + row + "," + col + " to " + value);
+			}
+			props[row][col] = (String) value;
+			fireTableCellUpdated(row, col);
+		}
+
+		private void printDebugData() {
+			int numRows = getRowCount();
+			int numCols = getColumnCount();
+
+			for (int i = 0; i < numRows; i++) {
+				System.out.print("    row " + i + ":");
+				for (int j = 0; j < numCols; j++) {
+					System.out.print("  " + props[i][j]);
+				}
+				System.out.println();
+			}
+			System.out.println("--------------------------");
+		}
+	}
+
+	class HelpButtonActionListener implements ActionListener {
+		public void actionPerformed(ActionEvent e) {
+			new Thread("DisplayHelpGUIThread") {
+				public void run() {
+
+				}
+			}.start();
+		}
+	}
 
-                }}.start();
-        }
-    }
-    
 }
-

marytts-builder/src/main/java/marytts/tools/voiceimport/SnackPitchmarker.java

@@ -34,270 +34,271 @@ import marytts.util.data.ESTTrackWriter;
 import marytts.util.data.text.SnackTextfileDoubleDataSource;
 import marytts.util.signal.SignalProcUtils;
 
-public class SnackPitchmarker extends VoiceImportComponent
-{
-    protected DatabaseLayout db = null;
-    //protected String correctedPmExt = ".pm.corrected";
-    protected String snackPmExt = ".snack";
-    protected String scriptFileName;
+public class SnackPitchmarker extends VoiceImportComponent {
+	protected DatabaseLayout db = null;
+	// protected String correctedPmExt = ".pm.corrected";
+	protected String snackPmExt = ".snack";
+	protected String scriptFileName;
 
-    private int percent = 0;
-  
-    public final String MINPITCH = "SnackPitchmarker.minPitch";
-    public final String MAXPITCH = "SnackPitchmarker.maxPitch";
-    public final String COMMAND = "SnackPitchmarker.command";
+	private int percent = 0;
 
-    public final String PMDIR = "db.pmDir";  
-    public final String PMEXT = "db.pmExtension";
-    
-    protected void setupHelp()
-    {
-        if (props2Help ==null){
-            props2Help = new TreeMap();
-            props2Help.put(MINPITCH,"minimum value for the pitch (in Hz). Default: female 100, male 75");
-            props2Help.put(MAXPITCH,"maximum value for the pitch (in Hz). Default: female 500, male 300"); 
-            props2Help.put(COMMAND, "the command that is used to launch Tcl; the Tcl installation must provide the SNACK package");
-        }
-    }
+	public final String MINPITCH = "SnackPitchmarker.minPitch";
+	public final String MAXPITCH = "SnackPitchmarker.maxPitch";
+	public final String COMMAND = "SnackPitchmarker.command";
 
-    public final String getName(){
-        return "SnackPitchmarker";
-    }
+	public final String PMDIR = "db.pmDir";
+	public final String PMEXT = "db.pmExtension";
 
-    @Override
-    protected void initialiseComp()
-    {
-        scriptFileName = db.getProp(db.TEMPDIR)+"pm.tcl";
-    }
+	protected void setupHelp() {
+		if (props2Help == null) {
+			props2Help = new TreeMap();
+			props2Help.put(MINPITCH, "minimum value for the pitch (in Hz). Default: female 100, male 75");
+			props2Help.put(MAXPITCH, "maximum value for the pitch (in Hz). Default: female 500, male 300");
+			props2Help
+					.put(COMMAND, "the command that is used to launch Tcl; the Tcl installation must provide the SNACK package");
+		}
+	}
 
-    public SortedMap getDefaultProps(DatabaseLayout db){
-        this.db = db;
-        if (props == null){
-            props = new TreeMap();       
-            if (db.getProp(db.GENDER).equals("female")){
-                props.put(MINPITCH,"100");
-                props.put(MAXPITCH,"500");
-            } else {
-                props.put(MINPITCH,"75");
-                props.put(MAXPITCH,"300");
-            }
-            if (MaryUtils.isWindows())
-                props.put(COMMAND, "c:/tcl/tclsh.exe"); // TODO someone with windows, please confirm or correct
-            else 
-                props.put(COMMAND, "/usr/bin/tclsh");
-            
-        }
-        return props;
-    }
+	public final String getName() {
+		return "SnackPitchmarker";
+	}
 
-    /**
-     * The standard compute() method of the VoiceImportComponent interface.
-     */
-    public boolean compute() throws Exception {
-        
-        File script = new File(scriptFileName);
-        
-        // What is the purpose of trunk/marytts/tools/voiceimport/pm.tcl, if it's hardcoded below here?
-        if (script.exists()) script.delete();
-        PrintWriter toScript = new PrintWriter(new FileWriter(script));
-        toScript.println("#!"+getProp(COMMAND));
-        toScript.println(" ");
-        toScript.println("package require snack");
-        toScript.println(" ");
-        toScript.println("snack::sound s");
-        toScript.println(" ");
-        toScript.println("s read [lindex $argv 0]");
-        toScript.println(" ");
-        toScript.println("set fd [open [lindex $argv 1] w]");
-        toScript.println("puts $fd [join [s pitch -method esps -maxpitch [lindex $argv 2] -minpitch [lindex $argv 3]] \\n]");
-        toScript.println("close $fd");
-        toScript.println(" ");
-        toScript.println("exit"); 
-        toScript.println(" ");
-        toScript.close();
+	@Override
+	protected void initialiseComp() {
+		scriptFileName = db.getProp(db.TEMPDIR) + "pm.tcl";
+	}
 
-        String[] baseNameArray = bnl.getListAsArray();
-        System.out.println( "Computing pitchmarks for " + baseNameArray.length + " utterances." );
+	public SortedMap getDefaultProps(DatabaseLayout db) {
+		this.db = db;
+		if (props == null) {
+			props = new TreeMap();
+			if (db.getProp(db.GENDER).equals("female")) {
+				props.put(MINPITCH, "100");
+				props.put(MAXPITCH, "500");
+			} else {
+				props.put(MINPITCH, "75");
+				props.put(MAXPITCH, "300");
+			}
+			if (MaryUtils.isWindows())
+				props.put(COMMAND, "c:/tcl/tclsh.exe"); // TODO someone with windows, please confirm or correct
+			else
+				props.put(COMMAND, "/usr/bin/tclsh");
 
-        /* Ensure the existence of the target pitchmark directory */
-        File dir = new File(db.getProp(PMDIR));
-        if (!dir.exists()) {
-            System.out.println( "Creating the directory [" + db.getProp(PMDIR) + "]." );
-            dir.mkdir();
-        }        
-        
-        /* execute snack */        
-        for ( int i = 0; i < baseNameArray.length; i++ ) {
-            percent = 100*i/baseNameArray.length;
-            String wavFile = db.getProp(db.WAVDIR) + baseNameArray[i] + db.getProp(db.WAVEXT);
-            String snackFile = db.getProp(PMDIR) + baseNameArray[i] + snackPmExt;
-            String pmFile = db.getProp(PMDIR) + baseNameArray[i] + db.getProp(PMEXT);
-            //String correctedPmFile = getProp(PMDIR) + baseNameArray[i] + correctedPmExt;
-            System.out.println("Writing pm file to "+snackFile);
+		}
+		return props;
+	}
 
-            boolean isWindows = true; // TODO This is WRONG, and never used. Consider removal.
-            String strTmp = scriptFileName + " " + wavFile + " " + snackFile + " " + getProp(MAXPITCH) + " " + getProp(MINPITCH);
+	/**
+	 * The standard compute() method of the VoiceImportComponent interface.
+	 */
+	public boolean compute() throws Exception {
 
-            if (MaryUtils.isWindows())
-                strTmp = "cmd.exe /c " + strTmp;
-            else  strTmp = getProp(COMMAND) + " " + strTmp;
-                
-            //System.out.println("strTmp: "+strTmp);
-            Process snack = Runtime.getRuntime().exec(strTmp);
+		File script = new File(scriptFileName);
 
-            StreamGobbler errorGobbler = new 
-            StreamGobbler(snack.getErrorStream(), "err");            
+		// What is the purpose of trunk/marytts/tools/voiceimport/pm.tcl, if it's hardcoded below here?
+		if (script.exists())
+			script.delete();
+		PrintWriter toScript = new PrintWriter(new FileWriter(script));
+		toScript.println("#!" + getProp(COMMAND));
+		toScript.println(" ");
+		toScript.println("package require snack");
+		toScript.println(" ");
+		toScript.println("snack::sound s");
+		toScript.println(" ");
+		toScript.println("s read [lindex $argv 0]");
+		toScript.println(" ");
+		toScript.println("set fd [open [lindex $argv 1] w]");
+		toScript.println("puts $fd [join [s pitch -method esps -maxpitch [lindex $argv 2] -minpitch [lindex $argv 3]] \\n]");
+		toScript.println("close $fd");
+		toScript.println(" ");
+		toScript.println("exit");
+		toScript.println(" ");
+		toScript.close();
 
-            //read from output stream
-            StreamGobbler outputGobbler = new 
-            StreamGobbler(snack.getInputStream(), "out");    
+		String[] baseNameArray = bnl.getListAsArray();
+		System.out.println("Computing pitchmarks for " + baseNameArray.length + " utterances.");
 
-            //start reading from the streams
-            errorGobbler.start();
-            outputGobbler.start();
+		/* Ensure the existence of the target pitchmark directory */
+		File dir = new File(db.getProp(PMDIR));
+		if (!dir.exists()) {
+			System.out.println("Creating the directory [" + db.getProp(PMDIR) + "].");
+			dir.mkdir();
+		}
 
-            //close everything down
-            snack.waitFor();
-            snack.exitValue();
+		/* execute snack */
+		for (int i = 0; i < baseNameArray.length; i++) {
+			percent = 100 * i / baseNameArray.length;
+			String wavFile = db.getProp(db.WAVDIR) + baseNameArray[i] + db.getProp(db.WAVEXT);
+			String snackFile = db.getProp(PMDIR) + baseNameArray[i] + snackPmExt;
+			String pmFile = db.getProp(PMDIR) + baseNameArray[i] + db.getProp(PMEXT);
+			// String correctedPmFile = getProp(PMDIR) + baseNameArray[i] + correctedPmExt;
+			System.out.println("Writing pm file to " + snackFile);
 
-            // Now convert the snack format into EST pm format
-            double[] pm = new SnackTextfileDoubleDataSource(new FileReader(snackFile)).getAllData();
+			boolean isWindows = true; // TODO This is WRONG, and never used. Consider removal.
+			String strTmp = scriptFileName + " " + wavFile + " " + snackFile + " " + getProp(MAXPITCH) + " " + getProp(MINPITCH);
 
-            WavReader wf = new WavReader(wavFile);
-            int sampleRate = wf.getSampleRate();
-            PitchMarks snackPitchmarker = SignalProcUtils.pitchContour2pitchMarks(pm,sampleRate,wf.getNumSamples(),0.0075,0.01,false,0);
-            int[] pitchmarkSamples = snackPitchmarker.pitchMarks; 
+			if (MaryUtils.isWindows())
+				strTmp = "cmd.exe /c " + strTmp;
+			else
+				strTmp = getProp(COMMAND) + " " + strTmp;
 
-            float[] pitchmarkSeconds = new float[pitchmarkSamples.length];
-            for (int j=0; j<pitchmarkSeconds.length;j++){
-                pitchmarkSeconds[j] = (float) pitchmarkSamples[j]/ (float) sampleRate; 
-            }
+			// System.out.println("strTmp: "+strTmp);
+			Process snack = Runtime.getRuntime().exec(strTmp);
 
-            new ESTTrackWriter(pitchmarkSeconds, null, "pitchmarks").doWriteAndClose(pmFile, false, false);
+			StreamGobbler errorGobbler = new StreamGobbler(snack.getErrorStream(), "err");
 
-            // And correct pitchmark locations
-            //pitchmarkSeconds = adjustPitchmarks(wf, pitchmarkSeconds);
-            //new ESTTrackWriter(pitchmarkSeconds, null, "pitchmarks").doWriteAndClose(correctedPmFile, false, false);
+			// read from output stream
+			StreamGobbler outputGobbler = new StreamGobbler(snack.getInputStream(), "out");
 
-        }
-        return true;
-    }
+			// start reading from the streams
+			errorGobbler.start();
+			outputGobbler.start();
 
-    /**
-     * Shift the pitchmarks to the closest peak.
-     */
-    private float[] shiftToClosestPeak( float[] pmIn, short[] w, int sampleRate ) {
+			// close everything down
+			snack.waitFor();
+			snack.exitValue();
 
-        final int HORIZON = 32; // <= number of samples to seek before and after the pitchmark
-        float[] pmOut = new float[pmIn.length];
+			// Now convert the snack format into EST pm format
+			double[] pm = new SnackTextfileDoubleDataSource(new FileReader(snackFile)).getAllData();
 
-        /* Browse the pitchmarks */
-        int pm = 0;
-        int pmwmax = w.length - 1;
-        int TO = 0;
-        int max = 0;
-        for ( int pi = 0; pi < pmIn.length; pi++ ) {
-            pm = (int)( pmIn[pi] * sampleRate );
-            // If the pitchmark goes out of the waveform (this sometimes
-            // happens with the last one due to rounding errors), just clip it.
-            if ( pm > pmwmax ) {
-                // If this was not the last pitchmark, there is a problem
-                if ( pi < (pmIn.length-1)) {
-                    throw new RuntimeException( "Some pitchmarks are located above the location of the last waveform sample !" );
-                }
-                // Else, if it was the last pitchmark, clip it:
-                pmOut[pi] = (float) ((double)(pmwmax) / (double)(sampleRate));
-            }
-            // Else, if the pitchmark is in the waveform:
-            else {
-                /* Seek the max of the wav samples around the pitchmark */
-                max = pm;
-                // - Back:
-                TO = (pm-HORIZON) < 0 ? 0 : (pm-HORIZON);
-                for ( int i = pm-1; i >= TO; i-- ) {
-                    if ( w[i] > w[max] ) max = i;
-                }
-                // - Forth:
-                TO = (pm+HORIZON+1) > w.length ? w.length : (pm+HORIZON+1);
-                for ( int i = pm+1; i < TO; i++ ) {
-                    if ( w[i] >= w[max] ) max = i;
-                }
-                /* Translate the pitchmark */
-                pmOut[pi] = (float) ( (double)(max) / (double)(sampleRate) );
-            }
-        }
+			WavReader wf = new WavReader(wavFile);
+			int sampleRate = wf.getSampleRate();
+			PitchMarks snackPitchmarker = SignalProcUtils.pitchContour2pitchMarks(pm, sampleRate, wf.getNumSamples(), 0.0075,
+					0.01, false, 0);
+			int[] pitchmarkSamples = snackPitchmarker.pitchMarks;
 
-        return pmOut;
-    }
+			float[] pitchmarkSeconds = new float[pitchmarkSamples.length];
+			for (int j = 0; j < pitchmarkSeconds.length; j++) {
+				pitchmarkSeconds[j] = (float) pitchmarkSamples[j] / (float) sampleRate;
+			}
 
-    /**
-     * Shift the pitchmarks to the previous zero crossing.
-     */
-    private float[] shiftToPreviousZero( float[] pmIn, short[] w, int sampleRate ) {
+			new ESTTrackWriter(pitchmarkSeconds, null, "pitchmarks").doWriteAndClose(pmFile, false, false);
 
-        final int HORIZON = 32; // <= number of samples to seek before the pitchmark
-        float[] pmOut = new float[pmIn.length];
+			// And correct pitchmark locations
+			// pitchmarkSeconds = adjustPitchmarks(wf, pitchmarkSeconds);
+			// new ESTTrackWriter(pitchmarkSeconds, null, "pitchmarks").doWriteAndClose(correctedPmFile, false, false);
 
-        /* Browse the pitchmarks */
-        int pm = 0;
-        int TO = 0;
-        int zero = 0;
-        for ( int pi = 0; pi < pmIn.length; pi++ ) {
-            pm = (int)( pmIn[pi] * sampleRate );
-            /* If the initial pitchmark is on a zero, don't shift the pitchmark. */
-            if ( w[pm] == 0 ) {
-                pmOut[pi] = pmIn[pi];
-                continue;
-            }
-            /* Else: */
-            /* Seek the zero crossing preceding the pitchmark */
-            TO = (pm-HORIZON) < 0 ? 0 : (pm-HORIZON);
-            for ( zero = pm; ( zero > TO ) && ( (w[zero]*w[zero+1]) > 0 ); zero-- );
-            /* If no zero crossing was found, don't move the pitchmark */
-            if ( (zero == TO) && ( (w[zero]*w[zero+1]) > 0 )  ) {
-                pmOut[pi] = pmIn[pi];
-            }
-            /* If a zero crossing was found, translate the pitchmark */
-            else {
-                pmOut[pi] = (float) ( (double)( (-w[zero]) <  w[zero+1] ?  zero : (zero+1) ) / (double)(sampleRate) );
-            }
-        }
+		}
+		return true;
+	}
 
-        return pmOut;
-    }
+	/**
+	 * Shift the pitchmarks to the closest peak.
+	 */
+	private float[] shiftToClosestPeak(float[] pmIn, short[] w, int sampleRate) {
 
-    /**
-     * Adjust pitchmark position to the zero crossing preceding the closest peak.
-     * @param basename basename of the corresponding wav file
-     * @param pitchmarks the input pitchmarks
-     * @return the adjusted pitchmarks
-     */
-    private float[] adjustPitchmarks( WavReader wf, float[] pitchmarks ) throws IOException
-    {
-        /* Load the wav file */
+		final int HORIZON = 32; // <= number of samples to seek before and after the pitchmark
+		float[] pmOut = new float[pmIn.length];
 
-        short[] w = wf.getSamples();
-        float[] pmOut = null;
-        try {
-            /* Shift to the closest peak */
-            pmOut = shiftToClosestPeak( pitchmarks, w, wf.getSampleRate() );
-            /* Shift to the zero immediately preceding the closest peak */
-            pmOut = shiftToPreviousZero( pmOut, w, wf.getSampleRate() );
-        } catch ( RuntimeException e ) {
-            throw new RuntimeException(e );
-        }
-        return pmOut;
-    }
+		/* Browse the pitchmarks */
+		int pm = 0;
+		int pmwmax = w.length - 1;
+		int TO = 0;
+		int max = 0;
+		for (int pi = 0; pi < pmIn.length; pi++) {
+			pm = (int) (pmIn[pi] * sampleRate);
+			// If the pitchmark goes out of the waveform (this sometimes
+			// happens with the last one due to rounding errors), just clip it.
+			if (pm > pmwmax) {
+				// If this was not the last pitchmark, there is a problem
+				if (pi < (pmIn.length - 1)) {
+					throw new RuntimeException("Some pitchmarks are located above the location of the last waveform sample !");
+				}
+				// Else, if it was the last pitchmark, clip it:
+				pmOut[pi] = (float) ((double) (pmwmax) / (double) (sampleRate));
+			}
+			// Else, if the pitchmark is in the waveform:
+			else {
+				/* Seek the max of the wav samples around the pitchmark */
+				max = pm;
+				// - Back:
+				TO = (pm - HORIZON) < 0 ? 0 : (pm - HORIZON);
+				for (int i = pm - 1; i >= TO; i--) {
+					if (w[i] > w[max])
+						max = i;
+				}
+				// - Forth:
+				TO = (pm + HORIZON + 1) > w.length ? w.length : (pm + HORIZON + 1);
+				for (int i = pm + 1; i < TO; i++) {
+					if (w[i] >= w[max])
+						max = i;
+				}
+				/* Translate the pitchmark */
+				pmOut[pi] = (float) ((double) (max) / (double) (sampleRate));
+			}
+		}
 
+		return pmOut;
+	}
 
-    /**
-     * Provide the progress of computation, in percent, or -1 if
-     * that feature is not implemented.
-     * @return -1 if not implemented, or an integer between 0 and 100.
-     */
-    public int getProgress()
-    {
-        return percent;
-    }
+	/**
+	 * Shift the pitchmarks to the previous zero crossing.
+	 */
+	private float[] shiftToPreviousZero(float[] pmIn, short[] w, int sampleRate) {
+
+		final int HORIZON = 32; // <= number of samples to seek before the pitchmark
+		float[] pmOut = new float[pmIn.length];
+
+		/* Browse the pitchmarks */
+		int pm = 0;
+		int TO = 0;
+		int zero = 0;
+		for (int pi = 0; pi < pmIn.length; pi++) {
+			pm = (int) (pmIn[pi] * sampleRate);
+			/* If the initial pitchmark is on a zero, don't shift the pitchmark. */
+			if (w[pm] == 0) {
+				pmOut[pi] = pmIn[pi];
+				continue;
+			}
+			/* Else: */
+			/* Seek the zero crossing preceding the pitchmark */
+			TO = (pm - HORIZON) < 0 ? 0 : (pm - HORIZON);
+			for (zero = pm; (zero > TO) && ((w[zero] * w[zero + 1]) > 0); zero--)
+				;
+			/* If no zero crossing was found, don't move the pitchmark */
+			if ((zero == TO) && ((w[zero] * w[zero + 1]) > 0)) {
+				pmOut[pi] = pmIn[pi];
+			}
+			/* If a zero crossing was found, translate the pitchmark */
+			else {
+				pmOut[pi] = (float) ((double) ((-w[zero]) < w[zero + 1] ? zero : (zero + 1)) / (double) (sampleRate));
+			}
+		}
+
+		return pmOut;
+	}
+
+	/**
+	 * Adjust pitchmark position to the zero crossing preceding the closest peak.
+	 * 
+	 * @param basename
+	 *            basename of the corresponding wav file
+	 * @param pitchmarks
+	 *            the input pitchmarks
+	 * @return the adjusted pitchmarks
+	 */
+	private float[] adjustPitchmarks(WavReader wf, float[] pitchmarks) throws IOException {
+		/* Load the wav file */
+
+		short[] w = wf.getSamples();
+		float[] pmOut = null;
+		try {
+			/* Shift to the closest peak */
+			pmOut = shiftToClosestPeak(pitchmarks, w, wf.getSampleRate());
+			/* Shift to the zero immediately preceding the closest peak */
+			pmOut = shiftToPreviousZero(pmOut, w, wf.getSampleRate());
+		} catch (RuntimeException e) {
+			throw new RuntimeException(e);
+		}
+		return pmOut;
+	}
+
+	/**
+	 * Provide the progress of computation, in percent, or -1 if that feature is not implemented.
+	 * 
+	 * @return -1 if not implemented, or an integer between 0 and 100.
+	 */
+	public int getProgress() {
+		return percent;
+	}
 }
-

marytts-builder/src/main/java/marytts/tools/voiceimport/TimelineWriter.java

@@ -41,300 +41,307 @@ import marytts.unitselection.data.TimelineReader;
 import marytts.util.data.Datagram;
 import marytts.util.data.MaryHeader;
 
-
-
 /**
- * The TimelineWriter class provides an interface to create or update
- * a Timeline data file in Mary format, and to feed new datagrams
- * to the timeline file.
+ * The TimelineWriter class provides an interface to create or update a Timeline data file in Mary format, and to feed new
+ * datagrams to the timeline file.
  * 
  * @author sacha, marc
  *
  */
-public class TimelineWriter  {
+public class TimelineWriter {
 
-    protected RandomAccessFile raf = null; // The file to read from
-    protected MaryHeader maryHdr = null;   // The standard Mary header
-    protected TimelineReader.ProcHeader procHdr = null;   // The processing info header
-    
-    protected TimelineReader.Index idx = null; // A global time index for the variable-sized datagrams
+	protected RandomAccessFile raf = null; // The file to read from
+	protected MaryHeader maryHdr = null; // The standard Mary header
+	protected TimelineReader.ProcHeader procHdr = null; // The processing info header
 
-    /* Some specific header fields: */
-    protected int sampleRate = 0;
-    protected long numDatagrams = 0;
-    
-    protected long datagramsBytePos = 0;
-    protected long timeIdxBytePos = 0;
+	protected TimelineReader.Index idx = null; // A global time index for the variable-sized datagrams
 
-    /* Pointers to navigate the file: */
-    protected long timePtr = 0; // A time pointer to keep track of the time position in the file
-    // Note: a file pointer, keeping track of the byte position in the file, is implicitely
-    //  maintained by the browsed RandomAccessFile.
+	/* Some specific header fields: */
+	protected int sampleRate = 0;
+	protected long numDatagrams = 0;
 
-    /****************/
-    /* DATA FIELDS  */
-    /****************/
-    private int idxInterval;
-    private long datagramZoneBytePos;
-    private Vector<TimelineReader.IdxField> indexData;
-    private long prevBytePos;
-    private long prevTimePos;
-    
-    
-    /****************/
-    /* CONSTRUCTORS */
-    /****************/
-    
-    /**
-     * Constructor to create a timeline.
-     * 
-     * @param fileName The file to read the timeline from.
-     * @param procHdrString the string to use as a processing header.
-     * @param reqSampleRate the sample rate requested to measure time in this timeline.
-     * @param setIdxIntervalInSeconds the interval between two index entries, in seconds
-     */
-    public TimelineWriter( String fileName, String procHdrString, int reqSampleRate, double setIdxIntervalInSeconds ) {
-        
-        /* Check the arguments */
-        if ( reqSampleRate <= 0 ) {
-            throw new RuntimeException( "The sample rate [" + reqSampleRate + "] can't be negative or null when creating a timeline." );
-        }
-        if ( setIdxIntervalInSeconds <= 0.0 ) {
-            throw new RuntimeException( "The index interval [" + setIdxIntervalInSeconds + "] can't be negative or null when creating a timeline." );
-        }
-        
-        /* Open the file */
-        try {
-            File fid = new File( fileName );
-            /* Check if the file exists and should be deleted first. */
-            if ( fid.exists() ) fid.delete();
-            /* open */
-            raf = new RandomAccessFile( fid, "rw" );
-        }
-        catch ( FileNotFoundException e ) {
-            throw new Error("Timeline file [" + fileName + "] was not found." );
-        }
-        catch ( SecurityException e ) {
-            throw new Error("You do not have read access to the file [" + fileName + "]." );
-        }
-        
-        /* Make a new header */
-        try {
-            /* Make a new Mary header and write it */
-            maryHdr = new MaryHeader( MaryHeader.TIMELINE );
-            maryHdr.writeTo( raf );
-            
-            /* Make a new processing header and write it */
-            procHdr = new TimelineReader.ProcHeader( procHdrString );
-            procHdr.dump( raf );
-            
-            /* Make/write the data header */
-            sampleRate = reqSampleRate;
-            raf.writeInt( sampleRate );
-            
-            numDatagrams = 0;
-            raf.writeLong( numDatagrams );
-            
-            /* Write the positions, with fake ones for the idx and basenames */
-            datagramsBytePos = getBytePointer() + 16; // +16: account for the 2 upcoming long fields datagramsBytePos and timeIdxBytePos
-            raf.writeLong( datagramsBytePos );
-            timeIdxBytePos = 0;
-            raf.writeLong( 0l );
-            
-            // Remember important facts for index creation
-            idxInterval = (int)Math.round( setIdxIntervalInSeconds * (double)sampleRate );
-            datagramZoneBytePos = datagramsBytePos;
-            indexData = new Vector<TimelineReader.IdxField>();
-            prevBytePos = datagramsBytePos;
-            prevTimePos = 0;
-            
-            /* Now we can output the datagrams. */
-            
-        }
-        catch ( IOException e ) {
-            throw new RuntimeException( "IOException caught when constructing a timeline writer on file [" + fileName + "]: ", e );
-        }
-    }
-    
-    
+	protected long datagramsBytePos = 0;
+	protected long timeIdxBytePos = 0;
 
-    
-    /*******************/
-    /* MISC. METHODS   */
-    /*******************/
-    
-    /**
-     * Get the current byte position in the file
-     */
-    public synchronized long getBytePointer() throws IOException {
-        return( raf.getFilePointer() );
-    }
-    
-    /**
-     * Get the current time position in the file
-     */
-    public synchronized long getTimePointer() {
-        return( timePtr );
-    }
-    
-    /**
-     * Set the current byte position in the file
-     */
-    protected void setBytePointer( long bytePos ) throws IOException {
-        raf.seek( bytePos );
-    }
-    
-    /**
-     * Set the current time position in the file
-     */
-    protected void setTimePointer( long timePosition ) {
-        timePtr = timePosition;
-    }
-    
-    /**
-     * Scales a discrete time to the timeline's sample rate.
-     * 
-     * @param reqSampleRate the externally given sample rate.
-     * @param targetTimeInSamples a discrete time, with respect to the externally given sample rate.
-     * 
-     * @return a discrete time, in samples with respect to the timeline's sample rate.
-     */
-    protected long scaleTime( int reqSampleRate, long targetTimeInSamples ) {
-        if ( reqSampleRate == sampleRate ) return( targetTimeInSamples );
-        /* else */ return( (long)Math.round( (double)(reqSampleRate) * (double)(targetTimeInSamples) / (double)(sampleRate) ) );
-    }
+	/* Pointers to navigate the file: */
+	protected long timePtr = 0; // A time pointer to keep track of the time position in the file
+	// Note: a file pointer, keeping track of the byte position in the file, is implicitely
+	// maintained by the browsed RandomAccessFile.
 
-    /**
-     * Unscales a discrete time from the timeline's sample rate.
-      * 
-     * @param reqSampleRate the externally given sample rate.
-     * @param timelineTimeInSamples a discrete time, with respect to the timeline sample rate.
-     * 
-     * @return a discrete time, in samples with respect to the externally given sample rate.
-    */
-    protected long unScaleTime( int reqSampleRate, long timelineTimeInSamples ) {
-        if ( reqSampleRate == sampleRate ) return( timelineTimeInSamples );
-        /* else */ return( (long)Math.round( (double)(sampleRate) * (double)(timelineTimeInSamples) / (double)(reqSampleRate) ) );
-    }
-    
-    
-    public TimelineReader.Index getIndex()
-    {
-        return idx;
-    }
-    
-    /**
-     * Returns the position of the datagram zone
-     */
-    public long getDatagramsBytePos() {
-        return datagramsBytePos;
-    }
-    
-    /**
-     * Returns the current number of datagrams in the timeline.
-     */
-    public long getNumDatagrams() {
-        return numDatagrams;
-    }
-    
-    /**
-     * Returns the sample rate of the timeline.
-     */
-    public int getSampleRate() {
-        return sampleRate;
-    }    
-    
-    /**
-     * Output the internally maintained indexes and close the file.
-     * 
-     * @throws IOException
-     */
-    public void close() throws IOException {
-        
-        /* Correct the number of datagrams */
-        setBytePointer( datagramsBytePos - 24l );
-        raf.writeLong( numDatagrams );
-        
-        /* Go to the end of the file and output the time index */
-        timeIdxBytePos = raf.length();
-        setBytePointer( timeIdxBytePos );
-        idx = new TimelineReader.Index(idxInterval, indexData);
-        idx.dump( raf );
-        
-        /* Register the index positions */
-        setBytePointer( datagramsBytePos - 8l );
-        raf.writeLong( timeIdxBytePos );
-        
-        /* Finally, close the random access file */
-        raf.close();
-    }
-    
-    
-    
-    /**
-     * Feeds a file position (in bytes) and a time position (in samples) from a timeline,
-     * and determines if a new index field is to be added.
-     * 
-     * @return the number of index fields after the feed.
-     */
-    private void feedIndex( long bytePosition, long timePosition ) {
-        /* Get the time associated with the yet to come index field */
-        long nextIdxTime = indexData.size() * (long)idxInterval;
-        /* If the current time position passes the next possible index field,
-         * register the PREVIOUS datagram position in the new index field */
-        while ( nextIdxTime < timePosition ) {
-//            System.out.println( "Hitting a new index at position\t[" + bytePosition + "," + timePosition + "]." );
-//            System.out.println( "The crossed index is [" + nextIdxTime + "]." );
-//            System.out.println( "The registered (previous) position is\t[" + prevBytePos + "," + prevTimePos + "]." );
-//            IdxField testField = (IdxField)field.elementAt(currentNumIdx-1);
-//            System.out.println( "The previously indexed position was\t[" + testField.bytePtr + "," + testField.timePtr + "]." );
-            
-            indexData.add( new TimelineReader.IdxField(prevBytePos,prevTimePos) );
-            nextIdxTime += idxInterval;
-        }
-        
-        /* Memorize the observed datagram position */
-        prevBytePos = bytePosition;
-        prevTimePos = timePosition;
-    }
+	/****************/
+	/* DATA FIELDS */
+	/****************/
+	private int idxInterval;
+	private long datagramZoneBytePos;
+	private Vector<TimelineReader.IdxField> indexData;
+	private long prevBytePos;
+	private long prevTimePos;
 
-    
-    /**
-     * Write one datagram to the timeline.
-     * 
-     * @param newDatagram the datagram to write.
-     * @param reqSampleTime the sample rate at which the datagram duration is expressed.
-     * 
-     * @throws IOException
-     */
-    public void feed( Datagram d, int reqSampleRate ) throws IOException {
-//        System.out.println( "Feeding datagram [ " + d.data.length + " , " + d.duration + " ] at pos ( "
-//                + getBytePointer() + " , " + getTimePointer() + " )" );
-        /* Filter the datagram through the index (to automatically add an index field if needed) */
-        feedIndex( getBytePointer(), getTimePointer() );
-        /* Check if the datagram needs resampling */
-        if ( reqSampleRate != sampleRate ) d.setDuration(scaleTime(reqSampleRate,d.getDuration()));
-        /* Then write the datagram on disk */
-        d.write( raf ); // This implicitely advances the bytePointer
-        /* Then advance various other pointers */
-        setTimePointer( getTimePointer() + d.getDuration() );
-        numDatagrams++;
-//        System.out.println( "Reached pos ( " + getBytePointer() + " , " + getTimePointer() + " )" );
-        
-    }
+	/****************/
+	/* CONSTRUCTORS */
+	/****************/
 
-    /**
-     * Write a series of datagrams to the timeline.
-     * 
-     * @param newDatagrams an array of datagrams.
-     * @param reqSampleTime the sample rate at which the datagram durations are expressed.
-     * 
-     * @throws IOException
-     */
-    public void feed( Datagram[] dArray, int reqSampleTime ) throws IOException {
-        for ( int i = 0; i < dArray.length; i++ ) {
-            feed( dArray[i], reqSampleTime );
-        }
-    }
+	/**
+	 * Constructor to create a timeline.
+	 * 
+	 * @param fileName
+	 *            The file to read the timeline from.
+	 * @param procHdrString
+	 *            the string to use as a processing header.
+	 * @param reqSampleRate
+	 *            the sample rate requested to measure time in this timeline.
+	 * @param setIdxIntervalInSeconds
+	 *            the interval between two index entries, in seconds
+	 */
+	public TimelineWriter(String fileName, String procHdrString, int reqSampleRate, double setIdxIntervalInSeconds) {
+
+		/* Check the arguments */
+		if (reqSampleRate <= 0) {
+			throw new RuntimeException("The sample rate [" + reqSampleRate
+					+ "] can't be negative or null when creating a timeline.");
+		}
+		if (setIdxIntervalInSeconds <= 0.0) {
+			throw new RuntimeException("The index interval [" + setIdxIntervalInSeconds
+					+ "] can't be negative or null when creating a timeline.");
+		}
+
+		/* Open the file */
+		try {
+			File fid = new File(fileName);
+			/* Check if the file exists and should be deleted first. */
+			if (fid.exists())
+				fid.delete();
+			/* open */
+			raf = new RandomAccessFile(fid, "rw");
+		} catch (FileNotFoundException e) {
+			throw new Error("Timeline file [" + fileName + "] was not found.");
+		} catch (SecurityException e) {
+			throw new Error("You do not have read access to the file [" + fileName + "].");
+		}
+
+		/* Make a new header */
+		try {
+			/* Make a new Mary header and write it */
+			maryHdr = new MaryHeader(MaryHeader.TIMELINE);
+			maryHdr.writeTo(raf);
+
+			/* Make a new processing header and write it */
+			procHdr = new TimelineReader.ProcHeader(procHdrString);
+			procHdr.dump(raf);
+
+			/* Make/write the data header */
+			sampleRate = reqSampleRate;
+			raf.writeInt(sampleRate);
+
+			numDatagrams = 0;
+			raf.writeLong(numDatagrams);
+
+			/* Write the positions, with fake ones for the idx and basenames */
+			datagramsBytePos = getBytePointer() + 16; // +16: account for the 2 upcoming long fields datagramsBytePos and
+														// timeIdxBytePos
+			raf.writeLong(datagramsBytePos);
+			timeIdxBytePos = 0;
+			raf.writeLong(0l);
+
+			// Remember important facts for index creation
+			idxInterval = (int) Math.round(setIdxIntervalInSeconds * (double) sampleRate);
+			datagramZoneBytePos = datagramsBytePos;
+			indexData = new Vector<TimelineReader.IdxField>();
+			prevBytePos = datagramsBytePos;
+			prevTimePos = 0;
+
+			/* Now we can output the datagrams. */
+
+		} catch (IOException e) {
+			throw new RuntimeException("IOException caught when constructing a timeline writer on file [" + fileName + "]: ", e);
+		}
+	}
+
+	/*******************/
+	/* MISC. METHODS */
+	/*******************/
+
+	/**
+	 * Get the current byte position in the file
+	 */
+	public synchronized long getBytePointer() throws IOException {
+		return (raf.getFilePointer());
+	}
+
+	/**
+	 * Get the current time position in the file
+	 */
+	public synchronized long getTimePointer() {
+		return (timePtr);
+	}
+
+	/**
+	 * Set the current byte position in the file
+	 */
+	protected void setBytePointer(long bytePos) throws IOException {
+		raf.seek(bytePos);
+	}
+
+	/**
+	 * Set the current time position in the file
+	 */
+	protected void setTimePointer(long timePosition) {
+		timePtr = timePosition;
+	}
+
+	/**
+	 * Scales a discrete time to the timeline's sample rate.
+	 * 
+	 * @param reqSampleRate
+	 *            the externally given sample rate.
+	 * @param targetTimeInSamples
+	 *            a discrete time, with respect to the externally given sample rate.
+	 * 
+	 * @return a discrete time, in samples with respect to the timeline's sample rate.
+	 */
+	protected long scaleTime(int reqSampleRate, long targetTimeInSamples) {
+		if (reqSampleRate == sampleRate)
+			return (targetTimeInSamples);
+		/* else */return ((long) Math.round((double) (reqSampleRate) * (double) (targetTimeInSamples) / (double) (sampleRate)));
+	}
+
+	/**
+	 * Unscales a discrete time from the timeline's sample rate.
+	 * 
+	 * @param reqSampleRate
+	 *            the externally given sample rate.
+	 * @param timelineTimeInSamples
+	 *            a discrete time, with respect to the timeline sample rate.
+	 * 
+	 * @return a discrete time, in samples with respect to the externally given sample rate.
+	 */
+	protected long unScaleTime(int reqSampleRate, long timelineTimeInSamples) {
+		if (reqSampleRate == sampleRate)
+			return (timelineTimeInSamples);
+		/* else */return ((long) Math.round((double) (sampleRate) * (double) (timelineTimeInSamples) / (double) (reqSampleRate)));
+	}
+
+	public TimelineReader.Index getIndex() {
+		return idx;
+	}
+
+	/**
+	 * Returns the position of the datagram zone
+	 */
+	public long getDatagramsBytePos() {
+		return datagramsBytePos;
+	}
+
+	/**
+	 * Returns the current number of datagrams in the timeline.
+	 */
+	public long getNumDatagrams() {
+		return numDatagrams;
+	}
+
+	/**
+	 * Returns the sample rate of the timeline.
+	 */
+	public int getSampleRate() {
+		return sampleRate;
+	}
+
+	/**
+	 * Output the internally maintained indexes and close the file.
+	 * 
+	 * @throws IOException
+	 */
+	public void close() throws IOException {
+
+		/* Correct the number of datagrams */
+		setBytePointer(datagramsBytePos - 24l);
+		raf.writeLong(numDatagrams);
+
+		/* Go to the end of the file and output the time index */
+		timeIdxBytePos = raf.length();
+		setBytePointer(timeIdxBytePos);
+		idx = new TimelineReader.Index(idxInterval, indexData);
+		idx.dump(raf);
+
+		/* Register the index positions */
+		setBytePointer(datagramsBytePos - 8l);
+		raf.writeLong(timeIdxBytePos);
+
+		/* Finally, close the random access file */
+		raf.close();
+	}
+
+	/**
+	 * Feeds a file position (in bytes) and a time position (in samples) from a timeline, and determines if a new index field is
+	 * to be added.
+	 * 
+	 * @return the number of index fields after the feed.
+	 */
+	private void feedIndex(long bytePosition, long timePosition) {
+		/* Get the time associated with the yet to come index field */
+		long nextIdxTime = indexData.size() * (long) idxInterval;
+		/*
+		 * If the current time position passes the next possible index field, register the PREVIOUS datagram position in the new
+		 * index field
+		 */
+		while (nextIdxTime < timePosition) {
+			// System.out.println( "Hitting a new index at position\t[" + bytePosition + "," + timePosition + "]." );
+			// System.out.println( "The crossed index is [" + nextIdxTime + "]." );
+			// System.out.println( "The registered (previous) position is\t[" + prevBytePos + "," + prevTimePos + "]." );
+			// IdxField testField = (IdxField)field.elementAt(currentNumIdx-1);
+			// System.out.println( "The previously indexed position was\t[" + testField.bytePtr + "," + testField.timePtr + "]."
+			// );
+
+			indexData.add(new TimelineReader.IdxField(prevBytePos, prevTimePos));
+			nextIdxTime += idxInterval;
+		}
+
+		/* Memorize the observed datagram position */
+		prevBytePos = bytePosition;
+		prevTimePos = timePosition;
+	}
+
+	/**
+	 * Write one datagram to the timeline.
+	 * 
+	 * @param newDatagram
+	 *            the datagram to write.
+	 * @param reqSampleTime
+	 *            the sample rate at which the datagram duration is expressed.
+	 * 
+	 * @throws IOException
+	 */
+	public void feed(Datagram d, int reqSampleRate) throws IOException {
+		// System.out.println( "Feeding datagram [ " + d.data.length + " , " + d.duration + " ] at pos ( "
+		// + getBytePointer() + " , " + getTimePointer() + " )" );
+		/* Filter the datagram through the index (to automatically add an index field if needed) */
+		feedIndex(getBytePointer(), getTimePointer());
+		/* Check if the datagram needs resampling */
+		if (reqSampleRate != sampleRate)
+			d.setDuration(scaleTime(reqSampleRate, d.getDuration()));
+		/* Then write the datagram on disk */
+		d.write(raf); // This implicitely advances the bytePointer
+		/* Then advance various other pointers */
+		setTimePointer(getTimePointer() + d.getDuration());
+		numDatagrams++;
+		// System.out.println( "Reached pos ( " + getBytePointer() + " , " + getTimePointer() + " )" );
+
+	}
+
+	/**
+	 * Write a series of datagrams to the timeline.
+	 * 
+	 * @param newDatagrams
+	 *            an array of datagrams.
+	 * @param reqSampleTime
+	 *            the sample rate at which the datagram durations are expressed.
+	 * 
+	 * @throws IOException
+	 */
+	public void feed(Datagram[] dArray, int reqSampleTime) throws IOException {
+		for (int i = 0; i < dArray.length; i++) {
+			feed(dArray[i], reqSampleTime);
+		}
+	}
 
 }

marytts-builder/src/main/java/marytts/tools/voiceimport/TranscriptionAligner.java

@@ -48,140 +48,125 @@ import org.w3c.dom.Document;
 import org.xml.sax.SAXException;
 
 public class TranscriptionAligner extends VoiceImportComponent {
-    
-    private DatabaseLayout db;
-    private String locale;
-    private int progress;
-    DocumentBuilderFactory dbf;
-    DocumentBuilder docBuilder;
-    TransformerFactory tFactory;
-    Transformer transformer;
-    File xmlOutDir;
-    
-    private marytts.tools.analysis.MaryTranscriptionAligner aligner;
-    
 
-    
+	private DatabaseLayout db;
+	private String locale;
+	private int progress;
+	DocumentBuilderFactory dbf;
+	DocumentBuilder docBuilder;
+	TransformerFactory tFactory;
+	Transformer transformer;
+	File xmlOutDir;
 
-    
-    public TranscriptionAligner() {
-    }
+	private marytts.tools.analysis.MaryTranscriptionAligner aligner;
 
-    public String getName() {
-        return "TranscriptionAligner";
-    }
-    
-    @Override
-    protected void initialiseComp()
-    throws ParserConfigurationException, IOException, SAXException, TransformerConfigurationException, MaryConfigurationException
-    {
-        aligner = new MaryTranscriptionAligner(db.getAllophoneSet());
-        aligner.SetEnsureInitialBoundary(true);
-        xmlOutDir = new File((String) db.getProp(db.ALLOPHONESDIR));
-        if (!xmlOutDir.exists()) {
-            xmlOutDir.mkdir();
-        }
-        // for parsing xml files
-        dbf = DocumentBuilderFactory.newInstance();
-        dbf.setNamespaceAware(true);
-        docBuilder = dbf.newDocumentBuilder();
-        
-        // for writing xml files
-        tFactory = TransformerFactory.newInstance();
-        transformer = tFactory.newTransformer();
-    }
-    
-    public SortedMap<String,String> getDefaultProps(DatabaseLayout theDb)
-    {
-        this.db = theDb;
-        locale = db.getProp(db.LOCALE);
-        if (props == null){
-            props = new TreeMap<String, String>();
-        }
-        return props;
-    }
-    
-    protected void setupHelp()
-    {
-        props2Help = new TreeMap<String, String>();
-    }
-    
- 
-    
-    public int getProgress()
-    {
-        return progress;
-    }
-    /**
-     * align and change automatic transcriptions to manually 
-     * corrected ones.
-     * 
-     * XML-Version: this changes mary xml-files (PHONEMISED)
-     * @throws Exception 
-     * @throws XPathExpressionException 
-     */
-    public boolean compute()
-    throws Exception
-    {
-        
-        System.out.println("traversing through " + bnl.getLength() + " files");
-       
-        String promptAllophonesDir = db.getProp(db.PROMPTALLOPHONESDIR);
-        for (int i=0; i<bnl.getLength(); i++) {
-            progress = 100*i/bnl.getLength();
-            System.out.println(bnl.getName(i));
-            alignTranscription(bnl.getName(i));
-        }
-                
-        return true;
-    }
-    
-    public void alignTranscription(String baseName) throws Exception{
-        String promptAllophonesDir = db.getProp(db.PROMPTALLOPHONESDIR);
-        File nextFile = new File(promptAllophonesDir
-                +System.getProperty("file.separator")
-                +baseName+".xml");
-        // get original xml file
-        Document doc = docBuilder.parse(nextFile);
+	public TranscriptionAligner() {
+	}
 
-        // open destination xml file
-        Writer docDest  = new OutputStreamWriter(new FileOutputStream(xmlOutDir.getAbsolutePath() + System.getProperty("file.separator")+nextFile.getName()), "UTF-8");
+	public String getName() {
+		return "TranscriptionAligner";
+	}
 
-        // open file with manual transcription that is to be aligned
-        String manTransString;
-        try{
+	@Override
+	protected void initialiseComp() throws ParserConfigurationException, IOException, SAXException,
+			TransformerConfigurationException, MaryConfigurationException {
+		aligner = new MaryTranscriptionAligner(db.getAllophoneSet());
+		aligner.SetEnsureInitialBoundary(true);
+		xmlOutDir = new File((String) db.getProp(db.ALLOPHONESDIR));
+		if (!xmlOutDir.exists()) {
+			xmlOutDir.mkdir();
+		}
+		// for parsing xml files
+		dbf = DocumentBuilderFactory.newInstance();
+		dbf.setNamespaceAware(true);
+		docBuilder = dbf.newDocumentBuilder();
 
-            String trfdir = db.getProp(db.LABDIR);
-            
-            String trfname = trfdir + 
-            nextFile.getName().substring(0, nextFile.getName().length() - 4) + ".lab";
-            
-            System.out.println(trfname);
-            
-            manTransString = MaryTranscriptionAligner.readLabelFile(aligner.getEntrySeparator(), aligner.getEnsureInitialBoundary(), trfname);
+		// for writing xml files
+		tFactory = TransformerFactory.newInstance();
+		transformer = tFactory.newTransformer();
+	}
 
-            // align transcriptions
-            aligner.alignXmlTranscriptions(doc, manTransString);            
-        } catch ( FileNotFoundException e ) {
-            // transform the unchanged xml-structure to a file
-            System.out.println("No manual transcription found, copy original ...");
-        }
-        
-        // write results to output
-        DOMSource source = new DOMSource( doc );
-        StreamResult output = new StreamResult(docDest);
-        transformer.transform(source, output);
-    }
-  
+	public SortedMap<String, String> getDefaultProps(DatabaseLayout theDb) {
+		this.db = theDb;
+		locale = db.getProp(db.LOCALE);
+		if (props == null) {
+			props = new TreeMap<String, String>();
+		}
+		return props;
+	}
 
-    /**
-     * @param args
-     */
-    public static void main(String[] args) throws Exception
-    {
-        VoiceImportComponent vic  =  new TranscriptionAligner();
-        DatabaseLayout db = new DatabaseLayout(vic);
-        vic.compute();
-    }
+	protected void setupHelp() {
+		props2Help = new TreeMap<String, String>();
+	}
+
+	public int getProgress() {
+		return progress;
+	}
+
+	/**
+	 * align and change automatic transcriptions to manually corrected ones.
+	 * 
+	 * XML-Version: this changes mary xml-files (PHONEMISED)
+	 * 
+	 * @throws Exception
+	 * @throws XPathExpressionException
+	 */
+	public boolean compute() throws Exception {
+
+		System.out.println("traversing through " + bnl.getLength() + " files");
+
+		String promptAllophonesDir = db.getProp(db.PROMPTALLOPHONESDIR);
+		for (int i = 0; i < bnl.getLength(); i++) {
+			progress = 100 * i / bnl.getLength();
+			System.out.println(bnl.getName(i));
+			alignTranscription(bnl.getName(i));
+		}
+
+		return true;
+	}
+
+	public void alignTranscription(String baseName) throws Exception {
+		String promptAllophonesDir = db.getProp(db.PROMPTALLOPHONESDIR);
+		File nextFile = new File(promptAllophonesDir + System.getProperty("file.separator") + baseName + ".xml");
+		// get original xml file
+		Document doc = docBuilder.parse(nextFile);
+
+		// open destination xml file
+		Writer docDest = new OutputStreamWriter(new FileOutputStream(xmlOutDir.getAbsolutePath()
+				+ System.getProperty("file.separator") + nextFile.getName()), "UTF-8");
+
+		// open file with manual transcription that is to be aligned
+		String manTransString;
+		try {
+
+			String trfdir = db.getProp(db.LABDIR);
+
+			String trfname = trfdir + nextFile.getName().substring(0, nextFile.getName().length() - 4) + ".lab";
+
+			System.out.println(trfname);
+
+			manTransString = MaryTranscriptionAligner.readLabelFile(aligner.getEntrySeparator(),
+					aligner.getEnsureInitialBoundary(), trfname);
+
+			// align transcriptions
+			aligner.alignXmlTranscriptions(doc, manTransString);
+		} catch (FileNotFoundException e) {
+			// transform the unchanged xml-structure to a file
+			System.out.println("No manual transcription found, copy original ...");
+		}
+
+		// write results to output
+		DOMSource source = new DOMSource(doc);
+		StreamResult output = new StreamResult(docDest);
+		transformer.transform(source, output);
+	}
+
+	/**
+	 * @param args
+	 */
+	public static void main(String[] args) throws Exception {
+		VoiceImportComponent vic = new TranscriptionAligner();
+		DatabaseLayout db = new DatabaseLayout(vic);
+		vic.compute();
+	}
 }
-

marytts-builder/src/main/java/marytts/tools/voiceimport/UnknownWordsFrequencyComputer.java

@@ -45,202 +45,181 @@ import org.w3c.dom.traversal.NodeFilter;
 import org.w3c.dom.traversal.NodeIterator;
 
 /**
- * For the given texts, compute unit features and align them
- * with the given unit labels.
+ * For the given texts, compute unit features and align them with the given unit labels.
+ * 
  * @author schroed
  *
  */
-public class UnknownWordsFrequencyComputer extends VoiceImportComponent
-{
-    protected File textDir;
-    protected File unitfeatureDir;
-    protected String featsExt = ".pfeats";
-    protected String locale;
-    protected MaryClient mary;
-    protected String maryInputType;
-    protected String maryOutputType;
-    
-    protected DatabaseLayout db = null;
-    protected int percent = 0;
-    
-    public String FEATUREDIR = "UnknownWordsFrequencyComputer.featureDir";
-    public String MARYSERVERHOST = "UnknownWordsFrequencyComputer.maryServerHost";
-    public String MARYSERVERPORT = "UnknownWordsFrequencyComputer.maryServerPort";
-       
-   
-    
-    public String getName(){
-        return "UnknownWordsFrequencyComputer";
-    }
-    
-    public static String getMaryXMLHeaderWithInitialBoundary(String locale)
-    {
-        return "<?xml version=\"1.0\" encoding=\"UTF-8\" ?>\n" +
-            "<maryxml version=\"0.4\"\n" +
-            "xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\"\n" +
-            "xmlns=\"http://mary.dfki.de/2002/MaryXML\"\n" +
-            "xml:lang=\"" + locale + "\">\n" +
-            "<boundary duration=\"100\"/>\n";
-        
-    }
-    
-    @Override
-    protected void initialiseComp()
-    {      
-        locale = db.getProp(db.LOCALE);   
-        
-        mary = null; // initialised only if needed   
-        unitfeatureDir = new File(getProp(FEATUREDIR));
-        if (!unitfeatureDir.exists()){
-            System.out.print(FEATUREDIR+" "+getProp(FEATUREDIR)
-                    +" does not exist; ");
-            if (!unitfeatureDir.mkdir()){
-                throw new Error("Could not create FEATUREDIR");
-            }
-            System.out.print("Created successfully.\n");
-        }    
-        
-        maryInputType = "RAWMARYXML";
-        maryOutputType = "PHONEMES";
-    }
-     
-     public SortedMap getDefaultProps(DatabaseLayout db){
-         this.db = db;
-         if (props == null){
-             props = new TreeMap();
-             props.put(FEATUREDIR, db.getProp(db.ROOTDIR)
-                     +"phonemisedXML"
-                     +System.getProperty("file.separator"));
-             props.put(MARYSERVERHOST,"localhost");
-             props.put(MARYSERVERPORT,"59125");
-         } 
-         return props;
-     }
-     
-     protected void setupHelp(){
-         props2Help = new TreeMap();
-         props2Help.put(FEATUREDIR, "directory containing the phone features." 
-                 +"Will be created if it does not exist");
-         props2Help.put(MARYSERVERHOST,"the host were the Mary server is running, default: \"localhost\"");
-         props2Help.put(MARYSERVERPORT,"the port were the Mary server is listening, default: \"59125\"");
-     }
-     
-     public MaryClient getMaryClient() throws IOException
-     {
-        if (mary == null) {
-            try{
-                mary = MaryClient.getMaryClient(new Address(getProp(MARYSERVERHOST), Integer.parseInt(getProp(MARYSERVERPORT))));        
-            } catch (IOException e){
-                throw new IOException("Could not connect to Maryserver at "
-                        +getProp(MARYSERVERHOST)+" "+getProp(MARYSERVERPORT));
-            }
-        }
-        return mary;
-    }
+public class UnknownWordsFrequencyComputer extends VoiceImportComponent {
+	protected File textDir;
+	protected File unitfeatureDir;
+	protected String featsExt = ".pfeats";
+	protected String locale;
+	protected MaryClient mary;
+	protected String maryInputType;
+	protected String maryOutputType;
 
-    public boolean compute() throws IOException, Exception
-    {
-        textDir = new File(db.getProp(db.TEXTDIR));
-        System.out.println( "Computing unit features for " + bnl.getLength() + " files" );
-        for (int i=0; i<bnl.getLength(); i++) {
-            percent = 100*i/bnl.getLength();
-            computeFeaturesFor( bnl.getName(i) );
-            //System.out.println( "    " + bnl.getName(i) );
-        }
-        bnl.write(db.getProp(db.ROOTDIR)+File.separator+"newbaselist.txt");
-        System.out.println("Finished computing the unit features.");
-        return true;
-    }
+	protected DatabaseLayout db = null;
+	protected int percent = 0;
 
-    public void computeFeaturesFor(String basename) throws IOException, Exception
-    {
-        String text;
-        Locale localVoice;
-        localVoice = MaryUtils.string2locale(locale);
-        
-        // First, test if there is a corresponding .rawmaryxml file in textdir:
-        File rawmaryxmlFile = new File(db.getProp(db.MARYXMLDIR)
-                				+ basename + db.getProp(db.MARYXMLEXT));
-        if (rawmaryxmlFile.exists()) {
-            text = FileUtils.getFileAsString(rawmaryxmlFile, "UTF-8");
-        } else {
-            text = getMaryXMLHeaderWithInitialBoundary(locale)
-                + FileUtils.getFileAsString(new File(db.getProp(db.TEXTDIR) 
-                        		+ basename + db.getProp(db.TEXTEXT)), "UTF-8")
-                + "</maryxml>";
-        }
-        File pfeatFile = new File( unitfeatureDir, basename + featsExt );
-        OutputStream os = new BufferedOutputStream(new FileOutputStream(pfeatFile));
-        MaryClient maryClient = getMaryClient();
-        /*Vector voices = maryClient.getVoices(localVoice);
-        MaryClient.Voice defaultVoice = (MaryClient.Voice) voices.firstElement();
-        String voiceName = defaultVoice.name();*/
-        //maryClient.process(text, maryInputType, maryOutputType, null, null, os);
-       
-        maryClient.process(text, maryInputType, maryOutputType, locale, null, "slt-arctic", os);
-        //maryClient.process(text, maryInputType, maryOutputType, null, "slt-arctic", os, timeout);
-        //maryClient.getOutputDataTypes().size()
-        //MaryData result = new MaryData(os);
-        
-        os.flush();    
-        os.close();
-        
-        //System.out.println(" TO STRING: "+new FileReader(pfeatFile).toString());
-        //BufferedReader bfr = new BufferedReader(new FileReader(pfeatFile)); 
-        String line;
-        MaryData d = new MaryData(MaryDataType.get("PHONEMISED_EN"), Locale.US);
-        //d.readFrom(new ByteArrayInputStream(os.toByteArray()));
-        d.readFrom(new FileReader(pfeatFile));
-       
-        //MaryData d = new MaryData(pfeatFile);
-        Document doc = d.getDocument();
-        //Document acoustparams = d.getDocument();
-              
-        //NodeIterator it = ((DocumentTraversal)acoustparams).createNodeIterator(acoustparams, NodeFilter.SHOW_ELEMENT,new NameNodeFilter(new String[]{MaryXML.TOKEN, MaryXML.BOUNDARY}),false);
-        NodeIterator it = ((DocumentTraversal)doc).
-        createNodeIterator(doc, NodeFilter.SHOW_ELEMENT,
-                           new NameNodeFilter(MaryXML.TOKEN), false);
-        
-        Element t = null;
-        while ((t = (Element) it.nextNode()) != null) {
-            if (t.hasAttribute("g2p_method")){
-                String g2p = t.getAttribute("g2p_method");
-                String nodeText = t.getTextContent().trim();
-                if(g2p.equals("rules")){// && nodeText.equals("!")){
-                    System.out.print(basename + " ----> " + nodeText);
-                    if(bnl.contains(basename))
-                        bnl.remove(basename);
-                    System.out.println(" SO removing basename: "+basename); 
-                    
-                }
-                
-                // System.out.println("G2P:"+t.getAttribute("g2p_method"));
-                //System.out.println("Text:"+t.getTextContent());
-            }
-        }
-        
-                 
-                
-        
-        /*while((line =bfr.readLine()) != null){
-            //boolean b = m.matches();
-            if(Pattern.matches("rules", line))
-                    System.out.println(basename + " LINE ---> " + line);
-             
-        }*/
-        //System.out.println(" TO STRING: "+line); 
-        
-    }
+	public String FEATUREDIR = "UnknownWordsFrequencyComputer.featureDir";
+	public String MARYSERVERHOST = "UnknownWordsFrequencyComputer.maryServerHost";
+	public String MARYSERVERPORT = "UnknownWordsFrequencyComputer.maryServerPort";
 
-    /**
-     * Provide the progress of computation, in percent, or -1 if
-     * that feature is not implemented.
-     * @return -1 if not implemented, or an integer between 0 and 100.
-     */
-    public int getProgress()
-    {
-        return percent;
-    }
+	public String getName() {
+		return "UnknownWordsFrequencyComputer";
+	}
+
+	public static String getMaryXMLHeaderWithInitialBoundary(String locale) {
+		return "<?xml version=\"1.0\" encoding=\"UTF-8\" ?>\n" + "<maryxml version=\"0.4\"\n"
+				+ "xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\"\n" + "xmlns=\"http://mary.dfki.de/2002/MaryXML\"\n"
+				+ "xml:lang=\"" + locale + "\">\n" + "<boundary duration=\"100\"/>\n";
+
+	}
+
+	@Override
+	protected void initialiseComp() {
+		locale = db.getProp(db.LOCALE);
+
+		mary = null; // initialised only if needed
+		unitfeatureDir = new File(getProp(FEATUREDIR));
+		if (!unitfeatureDir.exists()) {
+			System.out.print(FEATUREDIR + " " + getProp(FEATUREDIR) + " does not exist; ");
+			if (!unitfeatureDir.mkdir()) {
+				throw new Error("Could not create FEATUREDIR");
+			}
+			System.out.print("Created successfully.\n");
+		}
+
+		maryInputType = "RAWMARYXML";
+		maryOutputType = "PHONEMES";
+	}
+
+	public SortedMap getDefaultProps(DatabaseLayout db) {
+		this.db = db;
+		if (props == null) {
+			props = new TreeMap();
+			props.put(FEATUREDIR, db.getProp(db.ROOTDIR) + "phonemisedXML" + System.getProperty("file.separator"));
+			props.put(MARYSERVERHOST, "localhost");
+			props.put(MARYSERVERPORT, "59125");
+		}
+		return props;
+	}
+
+	protected void setupHelp() {
+		props2Help = new TreeMap();
+		props2Help.put(FEATUREDIR, "directory containing the phone features." + "Will be created if it does not exist");
+		props2Help.put(MARYSERVERHOST, "the host were the Mary server is running, default: \"localhost\"");
+		props2Help.put(MARYSERVERPORT, "the port were the Mary server is listening, default: \"59125\"");
+	}
+
+	public MaryClient getMaryClient() throws IOException {
+		if (mary == null) {
+			try {
+				mary = MaryClient.getMaryClient(new Address(getProp(MARYSERVERHOST), Integer.parseInt(getProp(MARYSERVERPORT))));
+			} catch (IOException e) {
+				throw new IOException("Could not connect to Maryserver at " + getProp(MARYSERVERHOST) + " "
+						+ getProp(MARYSERVERPORT));
+			}
+		}
+		return mary;
+	}
+
+	public boolean compute() throws IOException, Exception {
+		textDir = new File(db.getProp(db.TEXTDIR));
+		System.out.println("Computing unit features for " + bnl.getLength() + " files");
+		for (int i = 0; i < bnl.getLength(); i++) {
+			percent = 100 * i / bnl.getLength();
+			computeFeaturesFor(bnl.getName(i));
+			// System.out.println( "    " + bnl.getName(i) );
+		}
+		bnl.write(db.getProp(db.ROOTDIR) + File.separator + "newbaselist.txt");
+		System.out.println("Finished computing the unit features.");
+		return true;
+	}
+
+	public void computeFeaturesFor(String basename) throws IOException, Exception {
+		String text;
+		Locale localVoice;
+		localVoice = MaryUtils.string2locale(locale);
+
+		// First, test if there is a corresponding .rawmaryxml file in textdir:
+		File rawmaryxmlFile = new File(db.getProp(db.MARYXMLDIR) + basename + db.getProp(db.MARYXMLEXT));
+		if (rawmaryxmlFile.exists()) {
+			text = FileUtils.getFileAsString(rawmaryxmlFile, "UTF-8");
+		} else {
+			text = getMaryXMLHeaderWithInitialBoundary(locale)
+					+ FileUtils.getFileAsString(new File(db.getProp(db.TEXTDIR) + basename + db.getProp(db.TEXTEXT)), "UTF-8")
+					+ "</maryxml>";
+		}
+		File pfeatFile = new File(unitfeatureDir, basename + featsExt);
+		OutputStream os = new BufferedOutputStream(new FileOutputStream(pfeatFile));
+		MaryClient maryClient = getMaryClient();
+		/*
+		 * Vector voices = maryClient.getVoices(localVoice); MaryClient.Voice defaultVoice = (MaryClient.Voice)
+		 * voices.firstElement(); String voiceName = defaultVoice.name();
+		 */
+		// maryClient.process(text, maryInputType, maryOutputType, null, null, os);
+
+		maryClient.process(text, maryInputType, maryOutputType, locale, null, "slt-arctic", os);
+		// maryClient.process(text, maryInputType, maryOutputType, null, "slt-arctic", os, timeout);
+		// maryClient.getOutputDataTypes().size()
+		// MaryData result = new MaryData(os);
+
+		os.flush();
+		os.close();
+
+		// System.out.println(" TO STRING: "+new FileReader(pfeatFile).toString());
+		// BufferedReader bfr = new BufferedReader(new FileReader(pfeatFile));
+		String line;
+		MaryData d = new MaryData(MaryDataType.get("PHONEMISED_EN"), Locale.US);
+		// d.readFrom(new ByteArrayInputStream(os.toByteArray()));
+		d.readFrom(new FileReader(pfeatFile));
+
+		// MaryData d = new MaryData(pfeatFile);
+		Document doc = d.getDocument();
+		// Document acoustparams = d.getDocument();
+
+		// NodeIterator it = ((DocumentTraversal)acoustparams).createNodeIterator(acoustparams, NodeFilter.SHOW_ELEMENT,new
+		// NameNodeFilter(new String[]{MaryXML.TOKEN, MaryXML.BOUNDARY}),false);
+		NodeIterator it = ((DocumentTraversal) doc).createNodeIterator(doc, NodeFilter.SHOW_ELEMENT, new NameNodeFilter(
+				MaryXML.TOKEN), false);
+
+		Element t = null;
+		while ((t = (Element) it.nextNode()) != null) {
+			if (t.hasAttribute("g2p_method")) {
+				String g2p = t.getAttribute("g2p_method");
+				String nodeText = t.getTextContent().trim();
+				if (g2p.equals("rules")) {// && nodeText.equals("!")){
+					System.out.print(basename + " ----> " + nodeText);
+					if (bnl.contains(basename))
+						bnl.remove(basename);
+					System.out.println(" SO removing basename: " + basename);
+
+				}
+
+				// System.out.println("G2P:"+t.getAttribute("g2p_method"));
+				// System.out.println("Text:"+t.getTextContent());
+			}
+		}
+
+		/*
+		 * while((line =bfr.readLine()) != null){ //boolean b = m.matches(); if(Pattern.matches("rules", line))
+		 * System.out.println(basename + " LINE ---> " + line);
+		 * 
+		 * }
+		 */
+		// System.out.println(" TO STRING: "+line);
+
+	}
+
+	/**
+	 * Provide the progress of computation, in percent, or -1 if that feature is not implemented.
+	 * 
+	 * @return -1 if not implemented, or an integer between 0 and 100.
+	 */
+	public int getProgress() {
+		return percent;
+	}
 
 }
-

marytts-builder/src/main/java/marytts/tools/voiceimport/VoiceCompiler.java

@@ -13,7 +13,6 @@ import java.util.SortedMap;
 import java.util.TreeMap;
 import java.util.regex.Pattern;
 
-
 import marytts.util.MaryUtils;
 import marytts.util.io.StreamGobbler;
 
@@ -28,47 +27,45 @@ import com.twmacinta.util.MD5;
  */
 public class VoiceCompiler extends VoiceImportComponent {
 
-	
-    // constants to access filenames in database component properties and organize file list:
+	// constants to access filenames in database component properties and organize file list:
 
-	public static  final String CARTFILE = "CARTBuilder.cartFile";
+	public static final String CARTFILE = "CARTBuilder.cartFile";
 
-	public static  final String DURTREE = "DurationCARTTrainer.durTree";
+	public static final String DURTREE = "DurationCARTTrainer.durTree";
 
-	public static  final String F0LEFTTREE = "F0CARTTrainer.f0LeftTreeFile";
+	public static final String F0LEFTTREE = "F0CARTTrainer.f0LeftTreeFile";
 
-	public static  final String F0MIDTREE = "F0CARTTrainer.f0MidTreeFile";
+	public static final String F0MIDTREE = "F0CARTTrainer.f0MidTreeFile";
 
-	public static  final String F0RIGHTTREE = "F0CARTTrainer.f0RightTreeFile";
+	public static final String F0RIGHTTREE = "F0CARTTrainer.f0RightTreeFile";
 
-	public static  final String HALFPHONEFEATSAC = "AcousticFeatureFileWriter.acFeatureFile";
+	public static final String HALFPHONEFEATSAC = "AcousticFeatureFileWriter.acFeatureFile";
 
-	public static  final String HALFPHONEFEATDEFAC = "AcousticFeatureFileWriter.acFeatDef";
+	public static final String HALFPHONEFEATDEFAC = "AcousticFeatureFileWriter.acFeatDef";
 
-	public static  final String HALFPHONEUNITS = "HalfPhoneUnitfileWriter.unitFile";
+	public static final String HALFPHONEUNITS = "HalfPhoneUnitfileWriter.unitFile";
 
-	public static  final String JOINCOSTFEATS = "JoinCostFileMaker.joinCostFile";
+	public static final String JOINCOSTFEATS = "JoinCostFileMaker.joinCostFile";
 
-	public static  final String JOINCOSTWEIGHTS = "JoinCostFileMaker.weightsFile";
+	public static final String JOINCOSTWEIGHTS = "JoinCostFileMaker.weightsFile";
 
-	public static  final String PHONEFEATDEF = "PhoneFeatureFileWriter.weightsFile";
+	public static final String PHONEFEATDEF = "PhoneFeatureFileWriter.weightsFile";
 
-    public static  final String WAVETIMELINE = "WaveTimelineMaker.waveTimeline";
+	public static final String WAVETIMELINE = "WaveTimelineMaker.waveTimeline";
 
-    public static  final String BASETIMELINE = "BasenameTimelineMaker.timelineFile";
+	public static final String BASETIMELINE = "BasenameTimelineMaker.timelineFile";
 
-	
 	protected MavenVoiceCompiler compiler;
-	
-	
+
 	/**
 	 * 
 	 */
 	public VoiceCompiler() {
 	}
-	
 
-	/* (non-Javadoc)
+	/*
+	 * (non-Javadoc)
+	 * 
 	 * @see marytts.tools.voiceimport.VoiceImportComponent#compute()
 	 */
 	@Override
@@ -76,56 +73,51 @@ public class VoiceCompiler extends VoiceImportComponent {
 
 		File compileDir = new File(getProp(getCompileDirProp()));
 		compiler = createCompiler(compileDir);
-		
+
 		if (!isUnitSelectionVoice()) {
 			mapFeatures();
 		}
-		
+
 		logger.info("Creating directories");
 		compiler.createDirectories();
-		
+
 		logger.info("Copying template files");
 		compiler.copyTemplateFiles();
-		
+
 		logger.info("Copying voice files");
 		compiler.copyVoiceFiles();
-		
+
 		logger.info("Compiling with Maven");
 		compiler.compileWithMaven();
-		
-		//logger.info("Creating component description file");
-		//compiler.createComponentFile();
+
+		// logger.info("Creating component description file");
+		// compiler.createComponentFile();
 		logger.info("done.");
-		
+
 		return true;
 	}
 
 	protected void mapFeatures() throws Exception {
-		throw new IllegalStateException("This method should not be called for unit selection voices, "+
-				" and hmm-based voices should extend it.");
+		throw new IllegalStateException("This method should not be called for unit selection voices, "
+				+ " and hmm-based voices should extend it.");
 	}
 
-
-
-
-
-	
 	protected boolean isUnitSelectionVoice() {
 		return true;
 	}
 
-
-
-	/* (non-Javadoc)
+	/*
+	 * (non-Javadoc)
+	 * 
 	 * @see marytts.tools.voiceimport.VoiceImportComponent#getDefaultProps(marytts.tools.voiceimport.DatabaseLayout)
 	 */
 	@Override
 	public SortedMap<String, String> getDefaultProps(DatabaseLayout db) {
-        if (props == null) {
-            props = new TreeMap<String, String>();
-            props.put(getCompileDirProp(), new File(db.getVoiceFileDir(), "voice-"+getVoiceName(db)).getAbsolutePath());
-        }
-        return props;
+		if (props == null) {
+			props = new TreeMap<String, String>();
+			props.put(getCompileDirProp(), new File(db.getVoiceFileDir(), "voice-" + getVoiceName(db)).getAbsolutePath());
+		}
+		return props;
 	}
 
 	protected String getVoiceName(DatabaseLayout db) {
@@ -136,7 +128,9 @@ public class VoiceCompiler extends VoiceImportComponent {
 		return "VoiceCompiler.compileDir";
 	}
 
-	/* (non-Javadoc)
+	/*
+	 * (non-Javadoc)
+	 * 
 	 * @see marytts.tools.voiceimport.VoiceImportComponent#getName()
 	 */
 	@Override
@@ -144,7 +138,9 @@ public class VoiceCompiler extends VoiceImportComponent {
 		return "VoiceCompiler";
 	}
 
-	/* (non-Javadoc)
+	/*
+	 * (non-Javadoc)
+	 * 
 	 * @see marytts.tools.voiceimport.VoiceImportComponent#getProgress()
 	 */
 	@Override
@@ -152,36 +148,35 @@ public class VoiceCompiler extends VoiceImportComponent {
 		return -1;
 	}
 
-	/* (non-Javadoc)
+	/*
+	 * (non-Javadoc)
+	 * 
 	 * @see marytts.tools.voiceimport.VoiceImportComponent#setupHelp()
 	 */
 	@Override
 	protected void setupHelp() {
-        props2Help = new TreeMap<String, String>();
-        props2Help.put(getCompileDirProp(), "The directory in which the files for compiling the voice will be copied.");
-    }
-	
+		props2Help = new TreeMap<String, String>();
+		props2Help.put(getCompileDirProp(), "The directory in which the files for compiling the voice will be copied.");
+	}
+
 	protected Map<String, String> getExtraVariableSubstitutionMap() {
 		return null;
 	}
-	
+
 	protected File[] getFilesForResources() {
-		String[] propsResources = new String[] {
-				CARTFILE, DURTREE, F0LEFTTREE, F0MIDTREE, F0RIGHTTREE, HALFPHONEFEATDEFAC, JOINCOSTWEIGHTS
-		};
+		String[] propsResources = new String[] { CARTFILE, DURTREE, F0LEFTTREE, F0MIDTREE, F0RIGHTTREE, HALFPHONEFEATDEFAC,
+				JOINCOSTWEIGHTS };
 		File[] filesForResources = new File[propsResources.length];
-		for (int i=0; i<propsResources.length; i++) {
+		for (int i = 0; i < propsResources.length; i++) {
 			filesForResources[i] = new File(db.getProperty(propsResources[i]));
 		}
 		return filesForResources;
 	}
-	
+
 	protected File[] getFilesForFilesystem() {
-		String[] propsFilesystem = new String[] {
-				HALFPHONEFEATSAC, HALFPHONEUNITS, JOINCOSTFEATS, BASETIMELINE, WAVETIMELINE	
-		};
+		String[] propsFilesystem = new String[] { HALFPHONEFEATSAC, HALFPHONEUNITS, JOINCOSTFEATS, BASETIMELINE, WAVETIMELINE };
 		File[] filesForFilesystem = new File[propsFilesystem.length];
-		for (int i=0; i<propsFilesystem.length; i++) {
+		for (int i = 0; i < propsFilesystem.length; i++) {
 			filesForFilesystem[i] = new File(db.getProperty(propsFilesystem[i]));
 		}
 		return filesForFilesystem;
@@ -191,12 +186,11 @@ public class VoiceCompiler extends VoiceImportComponent {
 		File[] filesForResources = getFilesForResources();
 		File[] filesForFilesystem = getFilesForFilesystem();
 		Map<String, String> extraVariablesToSubstitute = getExtraVariableSubstitutionMap();
-		return new MavenVoiceCompiler(compileDir, getVoiceName(db), db.getMaryVersion(), db.getLocale(), db.getGender(), db.getDomain(), db.getSamplingRate(), isUnitSelectionVoice(),
-				filesForResources, filesForFilesystem, extraVariablesToSubstitute);
+		return new MavenVoiceCompiler(compileDir, getVoiceName(db), db.getMaryVersion(), db.getLocale(), db.getGender(),
+				db.getDomain(), db.getSamplingRate(), isUnitSelectionVoice(), filesForResources, filesForFilesystem,
+				extraVariablesToSubstitute);
 	}
 
-
-	
 	public static class MavenVoiceCompiler {
 		protected File compileDir;
 		protected String voiceName;
@@ -209,7 +203,7 @@ public class VoiceCompiler extends VoiceImportComponent {
 		protected File[] filesForResources;
 		protected File[] filesForFilesystem;
 		protected Map<String, String> extraVariablesToSubstitute;
-		
+
 		protected StrSubstitutor substitutor;
 		protected File mainJavaDir;
 		protected File mainResourcesDir;
@@ -218,9 +212,10 @@ public class VoiceCompiler extends VoiceImportComponent {
 		protected File metaInfDir;
 		protected File testJavaDir;
 		protected File libVoiceDir;
-		
-		public MavenVoiceCompiler(File compileDir, String voiceName, String voiceVersion, Locale locale, String gender, String domain, int samplingRate, boolean isUnitSelectionVoice,
-				File[] filesForResources, File[] filesForFilesystem, Map<String, String> extraVariablesToSubstitute) {
+
+		public MavenVoiceCompiler(File compileDir, String voiceName, String voiceVersion, Locale locale, String gender,
+				String domain, int samplingRate, boolean isUnitSelectionVoice, File[] filesForResources,
+				File[] filesForFilesystem, Map<String, String> extraVariablesToSubstitute) {
 			this.compileDir = compileDir;
 			this.voiceName = voiceName;
 			this.voiceVersion = voiceVersion;
@@ -230,11 +225,11 @@ public class VoiceCompiler extends VoiceImportComponent {
 			this.samplingRate = samplingRate;
 			this.isUnitSelectionVoice = isUnitSelectionVoice;
 			this.substitutor = new StrSubstitutor(getVariableSubstitutionMap(extraVariablesToSubstitute));
-			
+
 			this.filesForResources = filesForResources;
 			this.filesForFilesystem = filesForFilesystem;
 		}
-		
+
 		protected Map<String, String> getVariableSubstitutionMap(Map<String, String> extra) {
 			Map<String, String> m = new HashMap<String, String>();
 			m.put("MARYVERSION", voiceVersion);
@@ -252,31 +247,30 @@ public class VoiceCompiler extends VoiceImportComponent {
 			return m;
 		}
 
-		
 		public void createDirectories() throws IOException {
-			 if (compileDir.exists()) {
-				 FileUtils.deleteDirectory(compileDir);
-			 }
-			 compileDir.mkdir();
-			 String packageName = getPackageName();
-			 mainJavaDir = new File(compileDir.getAbsolutePath()+"/src/main/java/marytts/voice/"+packageName);
-			 mainJavaDir.mkdirs();
-			 mainResourcesDir = new File(compileDir.getAbsolutePath()+"/src/main/resources/marytts/voice/"+packageName);
-			 mainResourcesDir.mkdirs();
-			 nonPackagedResourcesDir = new File(compileDir.getAbsolutePath()+"/src/non-packaged-resources");
-			 nonPackagedResourcesDir.mkdirs();
-			 mainDescriptionsDir = new File(compileDir.getAbsolutePath()+"/src/main/descriptors");
-			 mainDescriptionsDir.mkdirs();
-			 metaInfDir = new File(compileDir.getAbsolutePath()+"/src/main/resources/META-INF/services");
-			 metaInfDir.mkdirs();
-			 testJavaDir = new File(compileDir.getAbsolutePath()+"/src/test/java/marytts/voice/"+packageName);
-			 testJavaDir.mkdirs();
-			 if (filesForFilesystem != null && filesForFilesystem.length > 0) {
-				 libVoiceDir = new File(compileDir.getAbsolutePath()+"/lib/voices/"+voiceName);
-				 libVoiceDir.mkdir();
-			 }
+			if (compileDir.exists()) {
+				FileUtils.deleteDirectory(compileDir);
+			}
+			compileDir.mkdir();
+			String packageName = getPackageName();
+			mainJavaDir = new File(compileDir.getAbsolutePath() + "/src/main/java/marytts/voice/" + packageName);
+			mainJavaDir.mkdirs();
+			mainResourcesDir = new File(compileDir.getAbsolutePath() + "/src/main/resources/marytts/voice/" + packageName);
+			mainResourcesDir.mkdirs();
+			nonPackagedResourcesDir = new File(compileDir.getAbsolutePath() + "/src/non-packaged-resources");
+			nonPackagedResourcesDir.mkdirs();
+			mainDescriptionsDir = new File(compileDir.getAbsolutePath() + "/src/main/descriptors");
+			mainDescriptionsDir.mkdirs();
+			metaInfDir = new File(compileDir.getAbsolutePath() + "/src/main/resources/META-INF/services");
+			metaInfDir.mkdirs();
+			testJavaDir = new File(compileDir.getAbsolutePath() + "/src/test/java/marytts/voice/" + packageName);
+			testJavaDir.mkdirs();
+			if (filesForFilesystem != null && filesForFilesystem.length > 0) {
+				libVoiceDir = new File(compileDir.getAbsolutePath() + "/lib/voices/" + voiceName);
+				libVoiceDir.mkdir();
+			}
 		}
-		
+
 		public void copyTemplateFiles() throws IOException {
 			copyWithVarSubstitution("pom.xml", new File(compileDir, "pom.xml"));
 			copyWithVarSubstitution("component.xml", new File(nonPackagedResourcesDir, "voice-component.xml"));
@@ -290,8 +284,8 @@ public class VoiceCompiler extends VoiceImportComponent {
 				copyWithVarSubstitution("hsmm-voice.config", getConfigFile());
 			}
 		}
-		
-		public void setFilesForResources(File [] filesForResources){
+
+		public void setFilesForResources(File[] filesForResources) {
 			this.filesForResources = filesForResources;
 		}
 
@@ -302,9 +296,11 @@ public class VoiceCompiler extends VoiceImportComponent {
 		public File getMainResourcesDir() {
 			return mainResourcesDir;
 		}
-		
-		private void copyWithVarSubstitution(String resourceName, File destination, StrSubstitutor... moreSubstitutors) throws IOException {
-			String resource = marytts.util.io.FileUtils.getStreamAsString(getClass().getResourceAsStream("templates/"+resourceName), "UTF-8");
+
+		private void copyWithVarSubstitution(String resourceName, File destination, StrSubstitutor... moreSubstitutors)
+				throws IOException {
+			String resource = marytts.util.io.FileUtils.getStreamAsString(
+					getClass().getResourceAsStream("templates/" + resourceName), "UTF-8");
 			String resourceWithReplacements = substitutor.replace(resource);
 			for (StrSubstitutor more : moreSubstitutors) {
 				resourceWithReplacements = more.replace(resourceWithReplacements);
@@ -328,10 +324,7 @@ public class VoiceCompiler extends VoiceImportComponent {
 				}
 			}
 		}
-		
-		
 
-		
 		public void compileWithMaven() throws IOException, InterruptedException {
 			Process maven = Runtime.getRuntime().exec("mvn verify", null, compileDir);
 			StreamGobbler merr = new StreamGobbler(maven.getErrorStream(), "maven err");
@@ -344,34 +337,29 @@ public class VoiceCompiler extends VoiceImportComponent {
 			}
 		}
 
-//		public void createComponentFile() throws IOException {
-//			String zipFileName = substitutor.replace("voice-${VOICENAME}-${MARYVERSION}.zip");
-//			File zipFile = new File(compileDir.getAbsolutePath()+"/target/"+zipFileName);
-//	        String zipFileMd5Hash = MD5.asHex(MD5.getHash(zipFile));
-//	        Map<String, String> compMap = new HashMap<String, String>();
-//	        compMap.put("MD5", zipFileMd5Hash);
-//	        compMap.put("FILESIZE", String.valueOf(zipFile.length()));
-//	        StrSubstitutor compSubst = new StrSubstitutor(compMap);
-//			String componentFileName = substitutor.replace("voice-${VOICENAME}-${MARYVERSION}-component.xml");
-//			File componentFile = new File(compileDir.getAbsolutePath()+"/target/"+componentFileName);
-//	        copyWithVarSubstitution("component.xml", componentFile, compSubst);
-//		}
-		
+		// public void createComponentFile() throws IOException {
+		// String zipFileName = substitutor.replace("voice-${VOICENAME}-${MARYVERSION}.zip");
+		// File zipFile = new File(compileDir.getAbsolutePath()+"/target/"+zipFileName);
+		// String zipFileMd5Hash = MD5.asHex(MD5.getHash(zipFile));
+		// Map<String, String> compMap = new HashMap<String, String>();
+		// compMap.put("MD5", zipFileMd5Hash);
+		// compMap.put("FILESIZE", String.valueOf(zipFile.length()));
+		// StrSubstitutor compSubst = new StrSubstitutor(compMap);
+		// String componentFileName = substitutor.replace("voice-${VOICENAME}-${MARYVERSION}-component.xml");
+		// File componentFile = new File(compileDir.getAbsolutePath()+"/target/"+componentFileName);
+		// copyWithVarSubstitution("component.xml", componentFile, compSubst);
+		// }
+
 		public String getPackageName() {
 			return toPackageName(voiceName);
 		}
 	}
-	
-	
-	
-	
-	
-	
+
 	/**
-	 * Convert an arbitrary string into a valid java package name, as follows:
-	 * - any characters that are not alphanumeric or underscore are deleted;
-	 * - if the first character after a deleted one is a letter, it is capitalised.
-	 * - if the first character is not a letter, we prepend a "V" for "voice".
+	 * Convert an arbitrary string into a valid java package name, as follows: - any characters that are not alphanumeric or
+	 * underscore are deleted; - if the first character after a deleted one is a letter, it is capitalised. - if the first
+	 * character is not a letter, we prepend a "V" for "voice".
+	 * 
 	 * @param voiceName
 	 * @return
 	 */
@@ -385,7 +373,7 @@ public class VoiceCompiler extends VoiceImportComponent {
 			if (part.isEmpty()) {
 				continue;
 			}
-			String firstChar =  part.substring(0, 1);
+			String firstChar = part.substring(0, 1);
 			if (Pattern.matches(regexLCLetter, firstChar)) {
 				result.append(firstChar.toUpperCase()).append(part.substring(1));
 			} else {
@@ -397,9 +385,5 @@ public class VoiceCompiler extends VoiceImportComponent {
 		}
 		return result.toString();
 	}
-	
-	
-	
-	
 
 }

marytts-builder/src/main/java/marytts/tools/voiceimport/VoiceImportComponent.java

@@ -29,125 +29,125 @@ import org.apache.log4j.Logger;
 
 /**
  * A component in the process of importing a voice into MARY format.
+ * 
  * @author Marc Schröder, Anna Hunecke
  *
  */
-public abstract class VoiceImportComponent
-{
-    protected SortedMap<String,String> props = null;
-    protected SortedMap<String,String> props2Help = null;
-    protected BasenameList bnl;
-    protected DatabaseLayout db;
-    protected Logger logger;
-    
-    protected VoiceImportComponent() {
-        if (!Logger.getRootLogger().getAllAppenders().hasMoreElements()) {
-            BasicConfigurator.configure();
-        }
-        logger = MaryUtils.getLogger(getName());
-    }
-    
-    protected abstract void setupHelp();
-    /**
-     * Initialise a voice import component:
-     * update values of local properties;
-     * setup help text for properties;
-     * call to component specific intialisation
-     * 
-     * @param db the database layout
-     * @param bnl the list of basenames
-     * @param props the map from properties to values
-     */
-    public final void initialise(DatabaseLayout db, BasenameList bnl, SortedMap<String,String> props)
-    throws Exception
-    {
-        //setupHelp(); this is now done by DatabaseLayout
-    	this.db = db;
-        this.props = props;
-        this.bnl = bnl;
-        initialiseComp();
-    }
-    
-    /**
-     * Initialise a voice import component:
-     * component specific initialisation;
-     * to be overwritten by subclasses
-     * 
-     * @param bnl the list of basenames
-     */
-    protected void initialiseComp()
-    throws Exception
-    {}
-        
-    
-    /**
-     * Get the map of properties2values
-     * containing the default values
-     * @return map of props2values
-     */
-    public abstract SortedMap<String,String> getDefaultProps(DatabaseLayout db);
-    
-    /**
-     * Get the value for a property
-     * @param prop the property name
-     * @return the value
-     */
-    public String getProp(String prop){
-        return props.get(prop);
-    }
-    
-    /**
-     * Set a property to a value
-     * @param prop the property
-     * @param value the value
-     */
-    public void setProp(String prop, String value){
-        props.put(prop,value);
-    }
-    
-    /**
-     * Get the name of this component
-     * @return the name
-     */
-    public abstract String getName();
-    
-    /**
-     * Do the computations required by this component.
-     * 
-     * @return true on success, false on failure
-     */
-    public abstract boolean compute() throws Exception;
-    
-    /**
-     * Provide the progress of computation, in percent, or -1 if
-     * that feature is not implemented.
-     * @return -1 if not implemented, or an integer between 0 and 100.
-     */
-    public abstract int getProgress();
-    
-    public String getHelpText(){
-        StringBuilder helpText = new StringBuilder();
-        helpText.append("<html>\n<head>\n<title>SETTINGS HELP</title>\n"
-                +"</head>\n<body>\n"
-                +"<h2>Settings help for component "+getName()+"</h2>\n<dl>\n");
-        try{
-        for (String key : props2Help.keySet()){
-            String value = (String) props2Help.get(key);
-            helpText.append("<dt><strong>"+key+"</strong></dt>\n"
-                    +"<dd>"+value+"</dd>\n");
-        }
-        
-        helpText.append("</dl>\n</body>\n</html>");
-        return helpText.toString();
-        } catch (NullPointerException npe){
-            npe.printStackTrace();
-            throw new Error("No help text for component "+getName());
-        }
-    }
-    
-    public String getHelpTextForProp(String propname){
-        return props2Help.get(propname);
-    }
-    
-}
+public abstract class VoiceImportComponent {
+	protected SortedMap<String, String> props = null;
+	protected SortedMap<String, String> props2Help = null;
+	protected BasenameList bnl;
+	protected DatabaseLayout db;
+	protected Logger logger;
 
+	protected VoiceImportComponent() {
+		if (!Logger.getRootLogger().getAllAppenders().hasMoreElements()) {
+			BasicConfigurator.configure();
+		}
+		logger = MaryUtils.getLogger(getName());
+	}
+
+	protected abstract void setupHelp();
+
+	/**
+	 * Initialise a voice import component: update values of local properties; setup help text for properties; call to component
+	 * specific intialisation
+	 * 
+	 * @param db
+	 *            the database layout
+	 * @param bnl
+	 *            the list of basenames
+	 * @param props
+	 *            the map from properties to values
+	 */
+	public final void initialise(DatabaseLayout db, BasenameList bnl, SortedMap<String, String> props) throws Exception {
+		// setupHelp(); this is now done by DatabaseLayout
+		this.db = db;
+		this.props = props;
+		this.bnl = bnl;
+		initialiseComp();
+	}
+
+	/**
+	 * Initialise a voice import component: component specific initialisation; to be overwritten by subclasses
+	 * 
+	 * @param bnl
+	 *            the list of basenames
+	 */
+	protected void initialiseComp() throws Exception {
+	}
+
+	/**
+	 * Get the map of properties2values containing the default values
+	 * 
+	 * @return map of props2values
+	 */
+	public abstract SortedMap<String, String> getDefaultProps(DatabaseLayout db);
+
+	/**
+	 * Get the value for a property
+	 * 
+	 * @param prop
+	 *            the property name
+	 * @return the value
+	 */
+	public String getProp(String prop) {
+		return props.get(prop);
+	}
+
+	/**
+	 * Set a property to a value
+	 * 
+	 * @param prop
+	 *            the property
+	 * @param value
+	 *            the value
+	 */
+	public void setProp(String prop, String value) {
+		props.put(prop, value);
+	}
+
+	/**
+	 * Get the name of this component
+	 * 
+	 * @return the name
+	 */
+	public abstract String getName();
+
+	/**
+	 * Do the computations required by this component.
+	 * 
+	 * @return true on success, false on failure
+	 */
+	public abstract boolean compute() throws Exception;
+
+	/**
+	 * Provide the progress of computation, in percent, or -1 if that feature is not implemented.
+	 * 
+	 * @return -1 if not implemented, or an integer between 0 and 100.
+	 */
+	public abstract int getProgress();
+
+	public String getHelpText() {
+		StringBuilder helpText = new StringBuilder();
+		helpText.append("<html>\n<head>\n<title>SETTINGS HELP</title>\n" + "</head>\n<body>\n"
+				+ "<h2>Settings help for component " + getName() + "</h2>\n<dl>\n");
+		try {
+			for (String key : props2Help.keySet()) {
+				String value = (String) props2Help.get(key);
+				helpText.append("<dt><strong>" + key + "</strong></dt>\n" + "<dd>" + value + "</dd>\n");
+			}
+
+			helpText.append("</dl>\n</body>\n</html>");
+			return helpText.toString();
+		} catch (NullPointerException npe) {
+			npe.printStackTrace();
+			throw new Error("No help text for component " + getName());
+		}
+	}
+
+	public String getHelpTextForProp(String propname) {
+		return props2Help.get(propname);
+	}
+
+}

marytts-builder/src/main/java/marytts/tools/voiceimport/VoiceQualityTimelineMaker.java

@@ -37,108 +37,108 @@ import marytts.unitselection.data.FloatArrayDatagram;
  */
 public class VoiceQualityTimelineMaker extends AbstractTimelineMaker {
 
-    /**
-     * get the name of this {@link VoiceImportComponent}
-     * 
-     * @return the name
-     */
-    @Override
-    public String getName() {
-        return "VoiceQualityTimelineMaker";
-    }
+	/**
+	 * get the name of this {@link VoiceImportComponent}
+	 * 
+	 * @return the name
+	 */
+	@Override
+	public String getName() {
+		return "VoiceQualityTimelineMaker";
+	}
 
-    /**
-     * {@inheritDoc}
-     */
-    @Override
-    public String getType() {
-        return "vq";
-    }
+	/**
+	 * {@inheritDoc}
+	 */
+	@Override
+	public String getType() {
+		return "vq";
+	}
 
-    /**
-     * load a {@link VoiceQualityDataFile}
-     * 
-     * @param file
-     *            to load
-     * @return the VoiceQualityDataFile
-     */
-    @Override
-    protected VoiceQualityDataFile loadDataFile(File file) {
-        return new VoiceQualityDataFile(file);
-    }
+	/**
+	 * load a {@link VoiceQualityDataFile}
+	 * 
+	 * @param file
+	 *            to load
+	 * @return the VoiceQualityDataFile
+	 */
+	@Override
+	protected VoiceQualityDataFile loadDataFile(File file) {
+		return new VoiceQualityDataFile(file);
+	}
 
-    /**
-     * {@inheritDoc}
-     * 
-     * @return header string
-     */
-    @Override
-    protected String getProcessingHeader() throws IOException {
-        Properties properties = new Properties();
-        properties.setProperty("paramNames", Arrays.toString(getParamNames()));
+	/**
+	 * {@inheritDoc}
+	 * 
+	 * @return header string
+	 */
+	@Override
+	protected String getProcessingHeader() throws IOException {
+		Properties properties = new Properties();
+		properties.setProperty("paramNames", Arrays.toString(getParamNames()));
 
-        // awkward code to Stringify differently than toString():
-        ByteArrayOutputStream stream = new ByteArrayOutputStream();
-        properties.store(stream, null);
-        String header = stream.toString("latin1");
-        return header;
-    }
+		// awkward code to Stringify differently than toString():
+		ByteArrayOutputStream stream = new ByteArrayOutputStream();
+		properties.store(stream, null);
+		String header = stream.toString("latin1");
+		return header;
+	}
 
-    /**
-     * get names of voice quality parameters as a String array
-     * 
-     * @return paramNames
-     */
-    public String[] getParamNames() {
-        // TODO this should preferably call an appropriate getter in VoiceQuality...
-        String[] paramNames = new String[] { "OQG", "GOG", "SKG", "RCG", "IC" };
-        return paramNames;
-    }
+	/**
+	 * get names of voice quality parameters as a String array
+	 * 
+	 * @return paramNames
+	 */
+	public String[] getParamNames() {
+		// TODO this should preferably call an appropriate getter in VoiceQuality...
+		String[] paramNames = new String[] { "OQG", "GOG", "SKG", "RCG", "IC" };
+		return paramNames;
+	}
 
-    /**
-     * Extension of {@link AbstractDataFile} for voice quality parameter files generated by {@link VoiceQuality#writeVqFile}
-     * 
-     * @author steiner
-     * 
-     */
-    public class VoiceQualityDataFile extends AbstractDataFile {
+	/**
+	 * Extension of {@link AbstractDataFile} for voice quality parameter files generated by {@link VoiceQuality#writeVqFile}
+	 * 
+	 * @author steiner
+	 * 
+	 */
+	public class VoiceQualityDataFile extends AbstractDataFile {
 
-        protected int numParams;
+		protected int numParams;
 
-        /**
-         * main constructor
-         * 
-         * @param file
-         *            to load
-         */
-        public VoiceQualityDataFile(File file) {
-            super(file);
-        }
+		/**
+		 * main constructor
+		 * 
+		 * @param file
+		 *            to load
+		 */
+		public VoiceQualityDataFile(File file) {
+			super(file);
+		}
 
-        /**
-         * load the file
-         */
-        @Override
-        protected void load(File file) {
-            // load the file as a VoiceQuality instance:
-            String fileName = file.getAbsolutePath();
-            VoiceQuality vq = new VoiceQuality(fileName);
-            sampleRate = vq.params.samplingRate;
-            frameSkip = vq.params.skipsize;
-            numFrames = vq.params.numfrm;
-            numParams = vq.params.dimension;
-            frameDuration = (int) (sampleRate * frameSkip);
+		/**
+		 * load the file
+		 */
+		@Override
+		protected void load(File file) {
+			// load the file as a VoiceQuality instance:
+			String fileName = file.getAbsolutePath();
+			VoiceQuality vq = new VoiceQuality(fileName);
+			sampleRate = vq.params.samplingRate;
+			frameSkip = vq.params.skipsize;
+			numFrames = vq.params.numfrm;
+			numParams = vq.params.dimension;
+			frameDuration = (int) (sampleRate * frameSkip);
 
-            // add data frames with values for each VQ parameter (as floats):
-            datagrams = new FloatArrayDatagram[numFrames];
-            for (int f = 0; f < numFrames; f++) {
-                float[] frameData = new float[numParams];
-                for (int p = 0; p < numParams; p++) {
-                    frameData[p] = (float) vq.vq[p][f];
-                }
-                FloatArrayDatagram datagram = new FloatArrayDatagram(frameDuration, frameData);
-                datagrams[f] = datagram;
-            }            
-        }
-    }
+			// add data frames with values for each VQ parameter (as floats):
+			datagrams = new FloatArrayDatagram[numFrames];
+			for (int f = 0; f < numFrames; f++) {
+				float[] frameData = new float[numParams];
+				for (int p = 0; p < numParams; p++) {
+					frameData[p] = (float) vq.vq[p][f];
+				}
+				FloatArrayDatagram datagram = new FloatArrayDatagram(frameDuration, frameData);
+				datagrams[f] = datagram;
+			}
+		}
+	}
 }

marytts-builder/src/main/java/marytts/tools/voiceimport/traintrees/ContourTreeInspector.java

@@ -120,5 +120,4 @@ public class ContourTreeInspector {
         
 
     }
-
 }

marytts-builder/src/main/java/marytts/tools/voiceimport/vocalizations/HNMFeatureFileWriter.java

@@ -51,177 +51,182 @@ import marytts.vocalizations.VocalizationUnitFileReader;
 
 /**
  * A component to extract and write HNM features of all vocalizations into a single file
+ * 
  * @author sathish
  *
  */
-public class HNMFeatureFileWriter extends VoiceImportComponent {    
+public class HNMFeatureFileWriter extends VoiceImportComponent {
 
-    private String waveExt = ".wav";
-    private String ptcExt = ".ptc";
-    private String hnmAnalysisFileExt = ".ana";    
-    private int progress = 0;    
-    protected DatabaseLayout db = null;
-    protected BasenameList bnlVocalizations;
+	private String waveExt = ".wav";
+	private String ptcExt = ".ptc";
+	private String hnmAnalysisFileExt = ".ana";
+	private int progress = 0;
+	protected DatabaseLayout db = null;
+	protected BasenameList bnlVocalizations;
 
-    private final String WAVEDIR = getName()+".vocalizationWaveDir";
-    private final String HNMANADIR = getName()+".hnmAnalysisDir";
-    private final String OUTHNMFILE = getName()+".hnmAnalysisTimelineFile";
-    private final String UNITFILE = getName()+".unitFile";
+	private final String WAVEDIR = getName() + ".vocalizationWaveDir";
+	private final String HNMANADIR = getName() + ".hnmAnalysisDir";
+	private final String OUTHNMFILE = getName() + ".hnmAnalysisTimelineFile";
+	private final String UNITFILE = getName() + ".unitFile";
 
-    private HntmSynthesizerParams synthesisParamsBeforeNoiseAnalysis;
-    private HntmAnalyzerParams analysisParams;
-    private PitchFileHeader f0Params;
+	private HntmSynthesizerParams synthesisParamsBeforeNoiseAnalysis;
+	private HntmAnalyzerParams analysisParams;
+	private PitchFileHeader f0Params;
 
-    @Override
-    public SortedMap<String, String> getDefaultProps(DatabaseLayout db) {
-        this.db = db;
-        if (props == null){
-            props = new TreeMap<String, String>();
-            props.put(UNITFILE,db.getProp(db.VOCALIZATIONSDIR)+File.separator+"files"+File.separator+"vocalization_units"+db.getProp(db.MARYEXT));
-            props.put(HNMANADIR,  db.getProp(db.VOCALIZATIONSDIR)+File.separator+"hna"+File.separator);
-            props.put(WAVEDIR, db.getProp(db.VOCALIZATIONSDIR)+File.separator+"wav");
-            props.put(OUTHNMFILE,db.getProp(db.VOCALIZATIONSDIR)+File.separator+"files"+File.separator+"vocalization_hnm_analysis"+db.getProp(db.MARYEXT));
-        }
-        return props;
-    }
+	@Override
+	public SortedMap<String, String> getDefaultProps(DatabaseLayout db) {
+		this.db = db;
+		if (props == null) {
+			props = new TreeMap<String, String>();
+			props.put(UNITFILE, db.getProp(db.VOCALIZATIONSDIR) + File.separator + "files" + File.separator
+					+ "vocalization_units" + db.getProp(db.MARYEXT));
+			props.put(HNMANADIR, db.getProp(db.VOCALIZATIONSDIR) + File.separator + "hna" + File.separator);
+			props.put(WAVEDIR, db.getProp(db.VOCALIZATIONSDIR) + File.separator + "wav");
+			props.put(OUTHNMFILE, db.getProp(db.VOCALIZATIONSDIR) + File.separator + "files" + File.separator
+					+ "vocalization_hnm_analysis" + db.getProp(db.MARYEXT));
+		}
+		return props;
+	}
 
-    @Override
-    protected void setupHelp() {
-        if (props2Help ==null){
-            props2Help = new TreeMap<String, String>();
-            props2Help.put(UNITFILE, "unit file representing all vocalizations");
-            props2Help.put(HNMANADIR,  "HNM features directory");
-            props2Help.put(WAVEDIR, "vocalization wave files directory");
-            props2Help.put(OUTHNMFILE,"a single file to write all HNM features");
-        }
-    }
+	@Override
+	protected void setupHelp() {
+		if (props2Help == null) {
+			props2Help = new TreeMap<String, String>();
+			props2Help.put(UNITFILE, "unit file representing all vocalizations");
+			props2Help.put(HNMANADIR, "HNM features directory");
+			props2Help.put(WAVEDIR, "vocalization wave files directory");
+			props2Help.put(OUTHNMFILE, "a single file to write all HNM features");
+		}
+	}
 
-    @Override
-    public boolean compute() throws UnsupportedAudioFileException, IOException, MaryConfigurationException {
+	@Override
+	public boolean compute() throws UnsupportedAudioFileException, IOException, MaryConfigurationException {
 
-        VocalizationUnitFileReader listenerUnits = new VocalizationUnitFileReader(getProp(UNITFILE));
+		VocalizationUnitFileReader listenerUnits = new VocalizationUnitFileReader(getProp(UNITFILE));
 
-        // sanity checker
-        if ( listenerUnits.getNumberOfUnits() != bnlVocalizations.getLength() ) {
-            throw new MaryConfigurationException("The number of vocalizations given in basename list and number of units in unit file should be matched");
-        }
+		// sanity checker
+		if (listenerUnits.getNumberOfUnits() != bnlVocalizations.getLength()) {
+			throw new MaryConfigurationException(
+					"The number of vocalizations given in basename list and number of units in unit file should be matched");
+		}
 
-        F0TrackerAutocorrelationHeuristic pitchDetector = new F0TrackerAutocorrelationHeuristic(f0Params);
-        HntmAnalyzer ha = new HntmAnalyzer();
-        DataOutputStream outStream = new DataOutputStream(new FileOutputStream(new File(getProp(OUTHNMFILE))));
-        writeHeaderTo(outStream);
-        outStream.writeInt(listenerUnits.getNumberOfUnits());
+		F0TrackerAutocorrelationHeuristic pitchDetector = new F0TrackerAutocorrelationHeuristic(f0Params);
+		HntmAnalyzer ha = new HntmAnalyzer();
+		DataOutputStream outStream = new DataOutputStream(new FileOutputStream(new File(getProp(OUTHNMFILE))));
+		writeHeaderTo(outStream);
+		outStream.writeInt(listenerUnits.getNumberOfUnits());
 
-        for( int i=0; i < bnlVocalizations.getLength(); i++ ) {
-            progress = i * 100 / bnlVocalizations.getLength();
-            String wavFile = getProp(WAVEDIR)+File.separator+bnlVocalizations.getName(i)+waveExt;
-            String pitchFile = getProp(HNMANADIR)+File.separator+bnlVocalizations.getName(i)+ptcExt;
-            String analysisResultsFile = getProp(HNMANADIR)+File.separator+bnlVocalizations.getName(i)+hnmAnalysisFileExt;
-            PitchReaderWriter f0 = pitchDetector.pitchAnalyzeWavFile(wavFile, pitchFile);
-            AudioInputStream inputAudio = AudioSystem.getAudioInputStream(new File(wavFile));
-            int samplingRate = (int)inputAudio.getFormat().getSampleRate();
-            AudioDoubleDataSource signal = new AudioDoubleDataSource(inputAudio);
-            double[] x = signal.getAllData();
-            x = MathUtils.multiply(x, 32768.0);
-            HntmSpeechSignal hnmSignal = ha.analyze(x, samplingRate, f0, null, analysisParams, synthesisParamsBeforeNoiseAnalysis, analysisResultsFile);
-            hnmSignal.write(outStream);
-        }
-        outStream.close();
+		for (int i = 0; i < bnlVocalizations.getLength(); i++) {
+			progress = i * 100 / bnlVocalizations.getLength();
+			String wavFile = getProp(WAVEDIR) + File.separator + bnlVocalizations.getName(i) + waveExt;
+			String pitchFile = getProp(HNMANADIR) + File.separator + bnlVocalizations.getName(i) + ptcExt;
+			String analysisResultsFile = getProp(HNMANADIR) + File.separator + bnlVocalizations.getName(i) + hnmAnalysisFileExt;
+			PitchReaderWriter f0 = pitchDetector.pitchAnalyzeWavFile(wavFile, pitchFile);
+			AudioInputStream inputAudio = AudioSystem.getAudioInputStream(new File(wavFile));
+			int samplingRate = (int) inputAudio.getFormat().getSampleRate();
+			AudioDoubleDataSource signal = new AudioDoubleDataSource(inputAudio);
+			double[] x = signal.getAllData();
+			x = MathUtils.multiply(x, 32768.0);
+			HntmSpeechSignal hnmSignal = ha.analyze(x, samplingRate, f0, null, analysisParams,
+					synthesisParamsBeforeNoiseAnalysis, analysisResultsFile);
+			hnmSignal.write(outStream);
+		}
+		outStream.close();
 
-        HNMFeatureFileReader tester = new HNMFeatureFileReader(getProp(OUTHNMFILE));
-        int unitsOnDisk = tester.getNumberOfUnits();
-        if (unitsOnDisk == listenerUnits.getNumberOfUnits()) {
-            System.out.println("Can read right number of units");
-            return true;
-        } else {
-            System.out.println("Read wrong number of units: "+unitsOnDisk);
-            return false;
-        }
-    }
+		HNMFeatureFileReader tester = new HNMFeatureFileReader(getProp(OUTHNMFILE));
+		int unitsOnDisk = tester.getNumberOfUnits();
+		if (unitsOnDisk == listenerUnits.getNumberOfUnits()) {
+			System.out.println("Can read right number of units");
+			return true;
+		} else {
+			System.out.println("Read wrong number of units: " + unitsOnDisk);
+			return false;
+		}
+	}
 
-    /**
-     * Initialize this component
-     */
-    @Override
-    protected void initialiseComp() throws Exception {
+	/**
+	 * Initialize this component
+	 */
+	@Override
+	protected void initialiseComp() throws Exception {
 
-        createDirectoryifNotExists(getProp(HNMANADIR));
+		createDirectoryifNotExists(getProp(HNMANADIR));
 
-        try {
-            String basenameFile = db.getProp(db.VOCALIZATIONSDIR)+File.separator+"basenames.lst";
-            if ( (new File(basenameFile)).exists() ) {
-                System.out.println("Loading basenames of vocalizations from '"+basenameFile+"' list...");
-                bnlVocalizations = new BasenameList(basenameFile);
-                System.out.println("Found "+bnlVocalizations.getLength()+ " vocalizations in basename list");
-            }
-            else {
-                String vocalWavDir = db.getProp(db.VOCALIZATIONSDIR)+File.separator+"wav";
-                System.out.println("Loading basenames of vocalizations from '"+vocalWavDir+"' directory...");
-                bnlVocalizations = new BasenameList(vocalWavDir, ".wav");
-                System.out.println("Found "+bnlVocalizations.getLength()+ " vocalizations in "+ vocalWavDir + " directory");
-            }
-        } catch (IOException e) {
-            throw new MaryConfigurationException("Problem with basename list "+e);
-        }
-        f0Params = new PitchFileHeader();
-        analysisParams = new HntmAnalyzerParams();
+		try {
+			String basenameFile = db.getProp(db.VOCALIZATIONSDIR) + File.separator + "basenames.lst";
+			if ((new File(basenameFile)).exists()) {
+				System.out.println("Loading basenames of vocalizations from '" + basenameFile + "' list...");
+				bnlVocalizations = new BasenameList(basenameFile);
+				System.out.println("Found " + bnlVocalizations.getLength() + " vocalizations in basename list");
+			} else {
+				String vocalWavDir = db.getProp(db.VOCALIZATIONSDIR) + File.separator + "wav";
+				System.out.println("Loading basenames of vocalizations from '" + vocalWavDir + "' directory...");
+				bnlVocalizations = new BasenameList(vocalWavDir, ".wav");
+				System.out.println("Found " + bnlVocalizations.getLength() + " vocalizations in " + vocalWavDir + " directory");
+			}
+		} catch (IOException e) {
+			throw new MaryConfigurationException("Problem with basename list " + e);
+		}
+		f0Params = new PitchFileHeader();
+		analysisParams = new HntmAnalyzerParams();
 
-        analysisParams.harmonicModel = HntmAnalyzerParams.HARMONICS_PLUS_NOISE;
-        analysisParams.noiseModel = HntmAnalyzerParams.WAVEFORM;
-        analysisParams.useHarmonicAmplitudesDirectly = true;
-        analysisParams.harmonicSynthesisMethodBeforeNoiseAnalysis = HntmSynthesizerParams.LINEAR_PHASE_INTERPOLATION;
-        analysisParams.regularizedCepstrumWarpingMethod = RegularizedCepstrumEstimator.REGULARIZED_CEPSTRUM_WITH_POST_MEL_WARPING;
+		analysisParams.harmonicModel = HntmAnalyzerParams.HARMONICS_PLUS_NOISE;
+		analysisParams.noiseModel = HntmAnalyzerParams.WAVEFORM;
+		analysisParams.useHarmonicAmplitudesDirectly = true;
+		analysisParams.harmonicSynthesisMethodBeforeNoiseAnalysis = HntmSynthesizerParams.LINEAR_PHASE_INTERPOLATION;
+		analysisParams.regularizedCepstrumWarpingMethod = RegularizedCepstrumEstimator.REGULARIZED_CEPSTRUM_WITH_POST_MEL_WARPING;
 
-        synthesisParamsBeforeNoiseAnalysis = new HntmSynthesizerParams();
-        synthesisParamsBeforeNoiseAnalysis.harmonicPartSynthesisMethod = HntmSynthesizerParams.LINEAR_PHASE_INTERPOLATION;        
-    }
+		synthesisParamsBeforeNoiseAnalysis = new HntmSynthesizerParams();
+		synthesisParamsBeforeNoiseAnalysis.harmonicPartSynthesisMethod = HntmSynthesizerParams.LINEAR_PHASE_INTERPOLATION;
+	}
 
+	/**
+	 * Create new directory if the directory doesn't exist
+	 * 
+	 * @param dirName
+	 * @throws Exception
+	 */
+	private void createDirectoryifNotExists(String dirName) throws Exception {
+		if (!(new File(dirName)).exists()) {
+			System.out.println(dirName + " directory does not exist; ");
+			if (!(new File(dirName)).mkdirs()) {
+				throw new Exception("Could not create directory " + dirName);
+			}
+			System.out.println("Created successfully.\n");
+		}
+	}
 
-    /**
-     * Create new directory if the directory doesn't exist 
-     * @param dirName
-     * @throws Exception
-     */
-    private void createDirectoryifNotExists(String dirName) throws Exception {
-        if (!(new File(dirName)).exists()) {
-            System.out.println( dirName+ " directory does not exist; ");
-            if (!(new File(dirName)).mkdirs()) {
-                throw new Exception("Could not create directory "+dirName);
-            }
-            System.out.println("Created successfully.\n");
-        }
-    }
+	/**
+	 * Write the header of this feature file to the given DataOutput
+	 * 
+	 * @param out
+	 * @throws IOException
+	 */
+	protected void writeHeaderTo(DataOutput out) throws IOException {
+		new MaryHeader(MaryHeader.LISTENERFEATS).writeTo(out);
+	}
 
-    /**
-     * Write the header of this feature file to the given DataOutput
-     * @param out
-     * @throws IOException
-     */
-    protected void writeHeaderTo(DataOutput out) throws IOException {
-        new MaryHeader(MaryHeader.LISTENERFEATS).writeTo(out);
-    }
+	/**
+	 * Return this voice import component name
+	 */
+	@Override
+	public String getName() {
+		return "HNMFeatureFileWriter";
+	}
 
-    /**
-     * Return this voice import component name
-     */
-    @Override
-    public String getName() {
-        return "HNMFeatureFileWriter";
-    }
+	/**
+	 * Return the progress of this component
+	 */
+	@Override
+	public int getProgress() {
+		return this.progress;
+	}
 
-    /**
-     * Return the progress of this component
-     */
-    @Override
-    public int getProgress() {
-        return this.progress;
-    }
+	/**
+	 * @param args
+	 */
+	public static void main(String[] args) {
+		// TODO Auto-generated method stub
 
-    /**
-     * @param args
-     */
-    public static void main(String[] args) {
-        // TODO Auto-generated method stub
-
-    }
+	}
 }

marytts-builder/src/main/java/marytts/tools/voiceimport/vocalizations/MLSAFeatureFileComputer.java

@@ -33,352 +33,365 @@ import marytts.util.io.FileUtils;
 import marytts.util.io.General;
 
 /**
- * MLSA feature files extractor for vocalizations. It extracts mgc features, strengths and logf0 features 
+ * MLSA feature files extractor for vocalizations. It extracts mgc features, strengths and logf0 features
+ * 
  * @author sathish
  *
  */
-public class MLSAFeatureFileComputer extends VoiceImportComponent {    
+public class MLSAFeatureFileComputer extends VoiceImportComponent {
 
-    private String waveExt = ".wav";
-    private String rawExt  = ".raw";
-    private String lf0Ext  = ".lf0";
-    private String strExt  = ".str";
-    private String mgcExt  = ".mgc";
-    private int progress = -1;
+	private String waveExt = ".wav";
+	private String rawExt = ".raw";
+	private String lf0Ext = ".lf0";
+	private String strExt = ".str";
+	private String mgcExt = ".mgc";
+	private int progress = -1;
 
-    // Default LF0 parameters
-    private int SAMPFREQ = 16000;
-    private int FRAMESHIFT = 80;
-    private int LOWERF0_VALUE=100;
-    private int UPPERF0_VALUE=500;
-    // Default MGC parameters
-    private int FRAMELEN = 400;
-    private int FFTLEN = 512;
-    private float FREQWARP = 0.42f;
-    private int MGCORDER = 24; //MGCORDER is actually 24 plus 1 include MGC[0]
-    // Default STR parameters
-    private int STRORDER = 5;
+	// Default LF0 parameters
+	private int SAMPFREQ = 16000;
+	private int FRAMESHIFT = 80;
+	private int LOWERF0_VALUE = 100;
+	private int UPPERF0_VALUE = 500;
+	// Default MGC parameters
+	private int FRAMELEN = 400;
+	private int FFTLEN = 512;
+	private float FREQWARP = 0.42f;
+	private int MGCORDER = 24; // MGCORDER is actually 24 plus 1 include MGC[0]
+	// Default STR parameters
+	private int STRORDER = 5;
 
-    protected DatabaseLayout db = null;
-    protected BasenameList bnlVocalizations;
+	protected DatabaseLayout db = null;
+	protected BasenameList bnlVocalizations;
 
-    private final String WAVEDIR = getName()+".vocalizationWaveDir";
-    private final String MLSADIR = getName()+".vocalizationMLSAFilesDir";
-    private final String RAWDIR  = getName()+".rawFilesDir";
-    private final String SCRIPTSDIR = getName()+".scriptsDir";
-    private final String TCLCOMMAND = getName()+".tclsh-commandlinePath";
-    private final String SOXCOMMAND = getName()+".sox-commandlinePath";
-    private final String SPTKPATH   = getName()+".SPTK-Path";
-    private final String LOWERLF0   = getName()+".LowerF0Value";
-    private final String UPPERLF0   = getName()+".UpperF0Value";
+	private final String WAVEDIR = getName() + ".vocalizationWaveDir";
+	private final String MLSADIR = getName() + ".vocalizationMLSAFilesDir";
+	private final String RAWDIR = getName() + ".rawFilesDir";
+	private final String SCRIPTSDIR = getName() + ".scriptsDir";
+	private final String TCLCOMMAND = getName() + ".tclsh-commandlinePath";
+	private final String SOXCOMMAND = getName() + ".sox-commandlinePath";
+	private final String SPTKPATH = getName() + ".SPTK-Path";
+	private final String LOWERLF0 = getName() + ".LowerF0Value";
+	private final String UPPERLF0 = getName() + ".UpperF0Value";
 
+	@Override
+	public SortedMap<String, String> getDefaultProps(DatabaseLayout db) {
+		this.db = db;
+		if (props == null) {
+			props = new TreeMap();
+			props.put(WAVEDIR, db.getProp(db.VOCALIZATIONSDIR) + File.separator + "wav");
+			props.put(SCRIPTSDIR, db.getProp(db.VOCALIZATIONSDIR) + File.separator + "scripts");
+			String mlsaDir = db.getProp(db.VOCALIZATIONSDIR) + "mlsa";
+			props.put(MLSADIR, mlsaDir);
+			props.put(RAWDIR, db.getProp(db.VOCALIZATIONSDIR) + File.separator + "raw");
+			props.put(TCLCOMMAND, "/usr/bin/tclsh");
+			props.put(SOXCOMMAND, "/usr/bin/sox");
+			props.put(SPTKPATH, "/usr/bin/");
+			props.put(LOWERLF0, LOWERF0_VALUE + "");
+			props.put(UPPERLF0, UPPERF0_VALUE + "");
+		}
+		return props;
+	}
 
-    @Override
-    public SortedMap<String, String> getDefaultProps(DatabaseLayout db) {
-        this.db = db;
-        if (props == null){
-            props = new TreeMap();
-            props.put(WAVEDIR, db.getProp(db.VOCALIZATIONSDIR)+File.separator+"wav");
-            props.put(SCRIPTSDIR, db.getProp(db.VOCALIZATIONSDIR)+File.separator+"scripts");
-            String mlsaDir = db.getProp(db.VOCALIZATIONSDIR)+"mlsa";
-            props.put(MLSADIR, mlsaDir);
-            props.put(RAWDIR, db.getProp(db.VOCALIZATIONSDIR)+File.separator+"raw");
-            props.put(TCLCOMMAND, "/usr/bin/tclsh");
-            props.put(SOXCOMMAND, "/usr/bin/sox");
-            props.put(SPTKPATH, "/usr/bin/");
-            props.put(LOWERLF0, LOWERF0_VALUE+"");
-            props.put(UPPERLF0, UPPERF0_VALUE+"");
-        }
-        return props;
-    }
+	@Override
+	protected void setupHelp() {
+		if (props2Help == null) {
+			props2Help = new TreeMap();
+			props2Help.put(WAVEDIR, "directory that contains vocalization wave files ");
+			props2Help.put(SCRIPTSDIR, "directory that contains external scripts used to compute MGC, LF0 and MGC features");
+			props2Help.put(MLSADIR, "mlsa features directory");
+			props2Help.put(RAWDIR, "raw files directory");
+			props2Help.put(TCLCOMMAND, "tcl executable command");
+			props2Help.put(SOXCOMMAND, "sox executable command");
+			props2Help.put(SPTKPATH, "Path that contains SPTK executables");
+			props2Help.put(LOWERLF0, "lowest pitch value specification");
+			props2Help.put(UPPERLF0, "highest pitch value specification");
+		}
 
-    @Override
-    protected void setupHelp() {
-        if (props2Help ==null){
-            props2Help = new TreeMap();
-            props2Help.put(WAVEDIR, "directory that contains vocalization wave files ");
-            props2Help.put(SCRIPTSDIR, "directory that contains external scripts used to compute MGC, LF0 and MGC features");
-            props2Help.put(MLSADIR, "mlsa features directory");
-            props2Help.put(RAWDIR, "raw files directory");
-            props2Help.put(TCLCOMMAND, "tcl executable command");
-            props2Help.put(SOXCOMMAND, "sox executable command");
-            props2Help.put(SPTKPATH, "Path that contains SPTK executables");
-            props2Help.put(LOWERLF0, "lowest pitch value specification");
-            props2Help.put(UPPERLF0, "highest pitch value specification");
-        }
+	}
 
-    }
+	/**
+	 * compute logf0, mgc, strength features
+	 */
+	@Override
+	public boolean compute() throws Exception {
 
-    /**
-     * compute logf0, mgc, strength features
-     */
-    @Override
-    public boolean compute() throws Exception {
+		copyFilesandScripts();
+		convertWAVE2RAW();
+		computeLF0Features();
+		computeMGCFeatures();
+		computeSTRFeatures();
 
-        copyFilesandScripts();
-        convertWAVE2RAW();
-        computeLF0Features();
-        computeMGCFeatures();
-        computeSTRFeatures();
+		return true;
+	}
 
-        return true;
-    }
+	/**
+	 * Initialize this component
+	 * 
+	 * @throws MaryConfigurationException
+	 *             if there is problem with basename list
+	 */
+	@Override
+	protected void initialiseComp() throws Exception {
 
-    /**
-     * Initialize this component
-     * @throws MaryConfigurationException if there is problem with basename list
-     */
-    @Override
-    protected void initialiseComp() throws Exception {
+		createDirectoryifNotExists(getProp(MLSADIR));
+		createDirectoryifNotExists(getProp(RAWDIR));
 
-        createDirectoryifNotExists(getProp(MLSADIR));
-        createDirectoryifNotExists(getProp(RAWDIR));
+		try {
+			String basenameFile = db.getProp(db.VOCALIZATIONSDIR) + File.separator + "basenames.lst";
+			if ((new File(basenameFile)).exists()) {
+				System.out.println("Loading basenames of vocalizations from '" + basenameFile + "' list...");
+				bnlVocalizations = new BasenameList(basenameFile);
+				System.out.println("Found " + bnlVocalizations.getLength() + " vocalizations in basename list");
+			} else {
+				String vocalWavDir = db.getProp(db.VOCALIZATIONSDIR) + File.separator + "wav";
+				System.out.println("Loading basenames of vocalizations from '" + vocalWavDir + "' directory...");
+				bnlVocalizations = new BasenameList(vocalWavDir, ".wav");
+				System.out.println("Found " + bnlVocalizations.getLength() + " vocalizations in " + vocalWavDir + " directory");
+			}
+		} catch (IOException e) {
+			throw new MaryConfigurationException("Problem with basename list " + e);
+		}
+	}
 
-        try {
-            String basenameFile = db.getProp(db.VOCALIZATIONSDIR)+File.separator+"basenames.lst";
-            if ( (new File(basenameFile)).exists() ) {
-                System.out.println("Loading basenames of vocalizations from '"+basenameFile+"' list...");
-                bnlVocalizations = new BasenameList(basenameFile);
-                System.out.println("Found "+bnlVocalizations.getLength()+ " vocalizations in basename list");
-            }
-            else {
-                String vocalWavDir = db.getProp(db.VOCALIZATIONSDIR)+File.separator+"wav";
-                System.out.println("Loading basenames of vocalizations from '"+vocalWavDir+"' directory...");
-                bnlVocalizations = new BasenameList(vocalWavDir, ".wav");
-                System.out.println("Found "+bnlVocalizations.getLength()+ " vocalizations in "+ vocalWavDir + " directory");
-            }
-        } catch (IOException e) {
-            throw new MaryConfigurationException("Problem with basename list "+e);
-        }
-    }
+	/**
+	 * copy required scripts and files for third-party software execution
+	 * 
+	 * @throws Exception
+	 *             if it can't copy files from source location
+	 */
+	private void copyFilesandScripts() throws Exception {
 
-    /**
-     * copy required scripts and files for third-party software execution
-     * @throws Exception if it can't copy files from source location
-     */
-    private void copyFilesandScripts() throws Exception {
+		String sep = System.getProperty("file.separator");
 
-        String sep = System.getProperty("file.separator");
+		createDirectoryifNotExists(getProp(SCRIPTSDIR));
+		createDirectoryifNotExists(db.getProp(db.ROOTDIR) + sep + "filters");
 
-        createDirectoryifNotExists(getProp(SCRIPTSDIR));
-        createDirectoryifNotExists(db.getProp(db.ROOTDIR)+sep+"filters");
+		String logF0ShellScript = getProp(this.SCRIPTSDIR) + sep + "get_f0.sh";
+		String logF0TCLScript = getProp(this.SCRIPTSDIR) + sep + "get_f0.tcl";
+		String mgcShellScript = getProp(this.SCRIPTSDIR) + sep + "get_mgc.sh";
+		String strShellScript = getProp(this.SCRIPTSDIR) + sep + "get_str.sh";
+		String strTCLScript = getProp(this.SCRIPTSDIR) + sep + "get_str.tcl";
+		String filterFile = db.getProp(db.ROOTDIR) + sep + "filters" + sep + "mix_excitation_filters.txt";
+		String sourceDir = db.getProp(db.MARYBASE) + sep + "lib" + sep + "external" + sep + "vocalizations";
 
-        String logF0ShellScript = getProp(this.SCRIPTSDIR) + sep + "get_f0.sh"; 
-        String logF0TCLScript = getProp(this.SCRIPTSDIR) + sep + "get_f0.tcl";
-        String mgcShellScript = getProp(this.SCRIPTSDIR) + sep + "get_mgc.sh"; 
-        String strShellScript = getProp(this.SCRIPTSDIR) + sep + "get_str.sh"; 
-        String strTCLScript = getProp(this.SCRIPTSDIR) + sep + "get_str.tcl";
-        String filterFile = db.getProp(db.ROOTDIR) + sep + "filters" + sep + "mix_excitation_filters.txt";
-        String sourceDir = db.getProp(db.MARYBASE) + sep + "lib" + sep + "external" + sep + "vocalizations";
+		if (!FileUtils.exists(logF0ShellScript)) {
+			String sourceScript = sourceDir + sep + "scripts" + sep + "get_f0.sh";
+			FileUtils.copy(sourceScript, logF0ShellScript);
+			(new File(logF0ShellScript)).setExecutable(true);
+		}
 
-        if ( !FileUtils.exists(logF0ShellScript) ) {
-            String sourceScript = sourceDir + sep + "scripts" + sep + "get_f0.sh";
-            FileUtils.copy(sourceScript, logF0ShellScript);
-            (new File(logF0ShellScript)).setExecutable(true);
-        }
+		if (!FileUtils.exists(logF0TCLScript)) {
+			String sourceScript = sourceDir + sep + "scripts" + sep + "get_f0.tcl";
+			FileUtils.copy(sourceScript, logF0TCLScript);
+			(new File(logF0TCLScript)).setExecutable(true);
+		}
 
-        if ( !FileUtils.exists(logF0TCLScript) ) {
-            String sourceScript = sourceDir + sep + "scripts" + sep + "get_f0.tcl";
-            FileUtils.copy(sourceScript, logF0TCLScript);
-            (new File(logF0TCLScript)).setExecutable(true);
-        }
+		if (!FileUtils.exists(strShellScript)) {
+			String sourceScript = sourceDir + sep + "scripts" + sep + "get_str.sh";
+			FileUtils.copy(sourceScript, strShellScript);
+			(new File(strShellScript)).setExecutable(true);
+		}
 
-        if ( !FileUtils.exists(strShellScript) ) {
-            String sourceScript = sourceDir + sep + "scripts" + sep + "get_str.sh";
-            FileUtils.copy(sourceScript, strShellScript);
-            (new File(strShellScript)).setExecutable(true);
-        }
+		if (!FileUtils.exists(strTCLScript)) {
+			String sourceScript = sourceDir + sep + "scripts" + sep + "get_str.tcl";
+			FileUtils.copy(sourceScript, strTCLScript);
+			(new File(strTCLScript)).setExecutable(true);
+		}
 
-        if ( !FileUtils.exists(strTCLScript) ) {
-            String sourceScript = sourceDir + sep + "scripts" + sep + "get_str.tcl";
-            FileUtils.copy(sourceScript, strTCLScript);
-            (new File(strTCLScript)).setExecutable(true);
-        }
+		if (!FileUtils.exists(mgcShellScript)) {
+			String sourceScript = sourceDir + sep + "scripts" + sep + "get_mgc.sh";
+			FileUtils.copy(sourceScript, mgcShellScript);
+			(new File(mgcShellScript)).setExecutable(true);
+		}
 
-        if ( !FileUtils.exists(mgcShellScript) ) {
-            String sourceScript = sourceDir + sep + "scripts" + sep + "get_mgc.sh";
-            FileUtils.copy(sourceScript, mgcShellScript);
-            (new File(mgcShellScript)).setExecutable(true);
-        }
+		if (!FileUtils.exists(filterFile)) {
+			String sourceScript = sourceDir + sep + "filters" + sep + "mix_excitation_filters.txt";
+			FileUtils.copy(sourceScript, filterFile);
+		}
+	}
 
-        if ( !FileUtils.exists(filterFile) ) {
-            String sourceScript = sourceDir + sep + "filters" + sep + "mix_excitation_filters.txt";
-            FileUtils.copy(sourceScript, filterFile);
-        }
-    }
+	/**
+	 * Convert all WAV files to RAW files to support SPTK
+	 * 
+	 * @throws IOException
+	 *             if can't run command
+	 * @throws ExecutionException
+	 *             if commandline throws an error
+	 */
+	private void convertWAVE2RAW() throws Exception {
 
-    /**
-     * Convert all WAV files to RAW files to support SPTK
-     * @throws IOException if can't run command
-     * @throws ExecutionException if commandline throws an error
-     */
-    private void convertWAVE2RAW() throws Exception {
+		for (int i = 0; i < bnlVocalizations.getLength(); i++) {
+			String waveFile = getProp(WAVEDIR) + File.separator + bnlVocalizations.getName(i) + waveExt;
+			String rawFile = getProp(RAWDIR) + File.separator + bnlVocalizations.getName(i) + rawExt;
+			String command = getProp(SOXCOMMAND) + " " + waveFile + " " + rawFile;
+			// System.out.println( "Command: " + command );
 
-        for ( int i=0; i < bnlVocalizations.getLength(); i++ ) {
-            String waveFile = getProp(WAVEDIR) + File.separator + bnlVocalizations.getName(i) + waveExt;
-            String rawFile  = getProp(RAWDIR) + File.separator + bnlVocalizations.getName(i) + rawExt;
-            String command = getProp(SOXCOMMAND) + " " +  waveFile + " " + rawFile;
-            //System.out.println( "Command: " + command );
+			try {
+				General.launchProc(command, "MLSAFeatureFileComputer", bnlVocalizations.getName(i));
+			} catch (Exception e) {
+				throw new ExecutionException("\nCommand failed : " + command + "\n" + e);
+			}
 
-            try {
-                General.launchProc(command, "MLSAFeatureFileComputer", bnlVocalizations.getName(i));
-            }
-            catch (Exception e) {
-                throw new ExecutionException("\nCommand failed : " + command + "\n" + e);
-            }
+			File rawEmptyFile = new File(rawFile);
+			if (rawEmptyFile.length() <= 0) { // delete all empty files
+				rawEmptyFile.delete();
+			}
+			System.out.println("Creating " + bnlVocalizations.getName(i) + " RAW file");
 
-            File rawEmptyFile = new File(rawFile);
-            if(rawEmptyFile.length() <= 0) { // delete all empty files
-                rawEmptyFile.delete();
-            }
-            System.out.println("Creating "+bnlVocalizations.getName(i)+" RAW file");
+			File rawNFile = new File(rawFile);
+			if (!rawNFile.exists()) {
+				throw new ExecutionException("The following command failed: \n " + command + "\n");
+			}
+		}
+	}
 
-            File rawNFile = new File(rawFile);
-            if ( !rawNFile.exists() ) {
-                throw new ExecutionException("The following command failed: \n " + command + "\n");
-            }
-        }
-    }
+	/**
+	 * Compute LF0 features for all RAW files using Snack and SPTK
+	 * 
+	 * @throws IOException
+	 *             if can't run command
+	 * @throws ExecutionException
+	 *             if commandline throws an error
+	 * @throws ExecutionException
+	 *             if there is no output file or output file is empty
+	 */
+	private void computeLF0Features() throws Exception {
+		String bcCommand = "/usr/bin/bc";
 
-    /**
-     * Compute LF0 features for all RAW files using Snack and SPTK
-     * @throws IOException if can't run command
-     * @throws ExecutionException if commandline throws an error
-     * @throws ExecutionException if there is no output file or output file is empty 
-     */
-    private void computeLF0Features() throws Exception {
-        String bcCommand = "/usr/bin/bc";
+		for (int i = 0; i < bnlVocalizations.getLength(); i++) {
+			String rawFile = getProp(RAWDIR) + File.separator + bnlVocalizations.getName(i) + rawExt;
+			String lf0File = getProp(MLSADIR) + File.separator + bnlVocalizations.getName(i) + lf0Ext;
+			String command = getProp(this.SCRIPTSDIR) + File.separator + "get_f0.sh " + bcCommand + " " + getProp(SPTKPATH) + " "
+					+ getProp(TCLCOMMAND) + " " + getProp(this.SCRIPTSDIR) + " " + rawFile + " " + lf0File + " " + SAMPFREQ + " "
+					+ FRAMESHIFT + " " + getProp(LOWERLF0) + " " + getProp(UPPERLF0);
+			try {
+				General.launchProc(command, "MLSAFeatureFileComputer", bnlVocalizations.getName(i));
+			} catch (Exception e) {
+				throw new ExecutionException("\nCommand failed : " + command + "\n" + e);
+			}
+			File lf0EmptyFile = new File(lf0File);
+			if (lf0EmptyFile.length() <= 0) { // delete all empty files
+				lf0EmptyFile.delete();
+			}
+			// System.out.println( "Command: " + command );
+			File lf0NFile = new File(lf0File);
+			if (!lf0NFile.exists()) {
+				throw new ExecutionException("The following command failed: \n " + command + "\n");
+			}
+			System.out.println("Computed LF0 features for " + bnlVocalizations.getName(i));
+		}
+	}
 
-        for ( int i=0; i < bnlVocalizations.getLength(); i++ ) {
-            String rawFile  = getProp(RAWDIR) + File.separator + bnlVocalizations.getName(i) + rawExt;
-            String lf0File  = getProp(MLSADIR) + File.separator + bnlVocalizations.getName(i) + lf0Ext;
-            String command = getProp(this.SCRIPTSDIR) + File.separator + "get_f0.sh " 
-                + bcCommand + " " + getProp(SPTKPATH) + " " + getProp(TCLCOMMAND) + " "
-                + getProp(this.SCRIPTSDIR) + " " + rawFile + " " + lf0File + " " 
-                + SAMPFREQ + " " + FRAMESHIFT + " " + getProp(LOWERLF0) + " " + getProp(UPPERLF0);
-            try {
-                General.launchProc(command, "MLSAFeatureFileComputer", bnlVocalizations.getName(i));
-            }
-            catch (Exception e) {
-                throw new ExecutionException("\nCommand failed : " + command + "\n" + e);
-            }
-            File lf0EmptyFile = new File(lf0File);
-            if(lf0EmptyFile.length() <= 0) { // delete all empty files
-                lf0EmptyFile.delete();
-            }
-            //System.out.println( "Command: " + command );
-            File lf0NFile = new File(lf0File);
-            if ( !lf0NFile.exists() ) {
-                throw new ExecutionException("The following command failed: \n " + command + "\n");
-            }
-            System.out.println("Computed LF0 features for "+bnlVocalizations.getName(i));
-        }
-    }
+	/**
+	 * Compute MGC features for all RAW files using SPTK
+	 * 
+	 * @throws IOException
+	 *             if can't run command
+	 * @throws ExecutionException
+	 *             if commandline throws an error
+	 * @throws ExecutionException
+	 *             if there is no output file or output file is empty
+	 */
+	private void computeMGCFeatures() throws Exception {
 
-    /**
-     * Compute MGC features for all RAW files using SPTK
-     * @throws IOException if can't run command
-     * @throws ExecutionException if commandline throws an error
-     * @throws ExecutionException if there is no output file or output file is empty 
-     */
-    private void computeMGCFeatures() throws Exception {
+		for (int i = 0; i < bnlVocalizations.getLength(); i++) {
+			String rawFile = getProp(RAWDIR) + File.separator + bnlVocalizations.getName(i) + rawExt;
+			String mgcFile = getProp(MLSADIR) + File.separator + bnlVocalizations.getName(i) + mgcExt;
+			String command = getProp(this.SCRIPTSDIR) + File.separator + "get_mgc.sh " + getProp(SPTKPATH) + " " + FRAMELEN + " "
+					+ FRAMESHIFT + " " + FFTLEN + " " + FREQWARP + " " + MGCORDER + " " + rawFile + " " + mgcFile;
+			try {
+				General.launchProc(command, "MLSAFeatureFileComputer", bnlVocalizations.getName(i));
+			} catch (Exception e) {
+				throw new ExecutionException("\nCommand failed : " + command + "\n" + e);
+			}
+			File lf0EmptyFile = new File(mgcFile);
+			if (lf0EmptyFile.length() <= 0) { // delete all empty files
+				lf0EmptyFile.delete();
+			}
+			// System.out.println( "Command: " + command );
 
-        for ( int i=0; i < bnlVocalizations.getLength(); i++ ) {
-            String rawFile  = getProp(RAWDIR) + File.separator + bnlVocalizations.getName(i) + rawExt;
-            String mgcFile  = getProp(MLSADIR) + File.separator + bnlVocalizations.getName(i) + mgcExt;
-            String command = getProp(this.SCRIPTSDIR) + File.separator + "get_mgc.sh " + getProp(SPTKPATH) + " "
-                + FRAMELEN + " " + FRAMESHIFT + " " + FFTLEN + " " + FREQWARP + " " + MGCORDER + " " + rawFile + " " + mgcFile;
-            try {
-                General.launchProc(command, "MLSAFeatureFileComputer", bnlVocalizations.getName(i));
-            }
-            catch (Exception e) {
-                throw new ExecutionException("\nCommand failed : " + command + "\n" + e);
-            }
-            File lf0EmptyFile = new File(mgcFile);
-            if(lf0EmptyFile.length() <= 0) { // delete all empty files
-                lf0EmptyFile.delete();
-            }
-            //System.out.println( "Command: " + command );
-            
-            File lf0NFile = new File(mgcFile);
-            if ( !lf0NFile.exists() ) {
-                throw new ExecutionException("The following command failed: \n " + command + "\n");
-            }
-            System.out.println("Computed MGC features for "+bnlVocalizations.getName(i));
-        }
-    }
+			File lf0NFile = new File(mgcFile);
+			if (!lf0NFile.exists()) {
+				throw new ExecutionException("The following command failed: \n " + command + "\n");
+			}
+			System.out.println("Computed MGC features for " + bnlVocalizations.getName(i));
+		}
+	}
 
-    /**
-     * Compute STRENGTH features for all RAW files using SPTK
-     * @throws IOException if can't run command
-     * @throws ExecutionException if commandline throws an error
-     * @throws ExecutionException if there is no output file or output file is empty 
-     */
-    private void computeSTRFeatures() throws Exception {
-        String bcCommand = "/usr/bin/bc";
+	/**
+	 * Compute STRENGTH features for all RAW files using SPTK
+	 * 
+	 * @throws IOException
+	 *             if can't run command
+	 * @throws ExecutionException
+	 *             if commandline throws an error
+	 * @throws ExecutionException
+	 *             if there is no output file or output file is empty
+	 */
+	private void computeSTRFeatures() throws Exception {
+		String bcCommand = "/usr/bin/bc";
 
-        for ( int i=0; i < bnlVocalizations.getLength(); i++ ) {
-            String rawFile  = getProp(RAWDIR) + File.separator + bnlVocalizations.getName(i) + rawExt;
-            String strFile  = getProp(MLSADIR) + File.separator + bnlVocalizations.getName(i) + strExt;
-            String command = getProp(this.SCRIPTSDIR) + File.separator + "get_str.sh " 
-                + bcCommand + " " + getProp(SPTKPATH) + " " + getProp(TCLCOMMAND) + " "
-                + getProp(this.SCRIPTSDIR) + " " + SAMPFREQ + " " + FRAMESHIFT + " " + getProp(LOWERLF0) + " " 
-                + getProp(UPPERLF0) + " " + STRORDER + " " + rawFile + " " + strFile + " ";
-            try {
-                General.launchProc(command, "MLSAFeatureFileComputer", bnlVocalizations.getName(i));
-            }
-            catch (Exception e) {
-                throw new ExecutionException("\nCommand failed : " + command + "\n" + e);
-            }
-            File lf0EmptyFile = new File(strFile);
-            if(lf0EmptyFile.length() <= 0) { // delete all empty files
-                lf0EmptyFile.delete();
-            }
-            //System.out.println( "Command: " + command );
-            File lf0NFile = new File(strFile);
-            if ( !lf0NFile.exists() ) {
-                throw new ExecutionException("The following command failed: \n " + command + "\n");
-            }
-            System.out.println("Computed Strength features for "+bnlVocalizations.getName(i));
-        }
-    }
+		for (int i = 0; i < bnlVocalizations.getLength(); i++) {
+			String rawFile = getProp(RAWDIR) + File.separator + bnlVocalizations.getName(i) + rawExt;
+			String strFile = getProp(MLSADIR) + File.separator + bnlVocalizations.getName(i) + strExt;
+			String command = getProp(this.SCRIPTSDIR) + File.separator + "get_str.sh " + bcCommand + " " + getProp(SPTKPATH)
+					+ " " + getProp(TCLCOMMAND) + " " + getProp(this.SCRIPTSDIR) + " " + SAMPFREQ + " " + FRAMESHIFT + " "
+					+ getProp(LOWERLF0) + " " + getProp(UPPERLF0) + " " + STRORDER + " " + rawFile + " " + strFile + " ";
+			try {
+				General.launchProc(command, "MLSAFeatureFileComputer", bnlVocalizations.getName(i));
+			} catch (Exception e) {
+				throw new ExecutionException("\nCommand failed : " + command + "\n" + e);
+			}
+			File lf0EmptyFile = new File(strFile);
+			if (lf0EmptyFile.length() <= 0) { // delete all empty files
+				lf0EmptyFile.delete();
+			}
+			// System.out.println( "Command: " + command );
+			File lf0NFile = new File(strFile);
+			if (!lf0NFile.exists()) {
+				throw new ExecutionException("The following command failed: \n " + command + "\n");
+			}
+			System.out.println("Computed Strength features for " + bnlVocalizations.getName(i));
+		}
+	}
 
-    /**
-     * Create new directory if the directory doesn't exist 
-     * @param dirName
-     * @throws Exception
-     */
-    private void createDirectoryifNotExists(String dirName) throws Exception {
-        if (!(new File(dirName)).exists()) {
-            System.out.println( dirName+ " directory does not exist; ");
-            if (!(new File(dirName)).mkdirs()) {
-                throw new Exception("Could not create directory "+dirName);
-            }
-            System.out.println("Created successfully.\n");
-        }
-    }
+	/**
+	 * Create new directory if the directory doesn't exist
+	 * 
+	 * @param dirName
+	 * @throws Exception
+	 */
+	private void createDirectoryifNotExists(String dirName) throws Exception {
+		if (!(new File(dirName)).exists()) {
+			System.out.println(dirName + " directory does not exist; ");
+			if (!(new File(dirName)).mkdirs()) {
+				throw new Exception("Could not create directory " + dirName);
+			}
+			System.out.println("Created successfully.\n");
+		}
+	}
 
-    /**
-     * Return this voice import component name
-     */
-    @Override
-    public String getName() {
-        return "MLSAFeatureFileComputer";
-    }
+	/**
+	 * Return this voice import component name
+	 */
+	@Override
+	public String getName() {
+		return "MLSAFeatureFileComputer";
+	}
 
-    /**
-     * Return the progress of this component
-     */
-    @Override
-    public int getProgress() {
-        return this.progress;
-    }
+	/**
+	 * Return the progress of this component
+	 */
+	@Override
+	public int getProgress() {
+		return this.progress;
+	}
 
-    /**
-     * @param args
-     */
-    public static void main(String[] args) {
-        // TODO Auto-generated method stub
+	/**
+	 * @param args
+	 */
+	public static void main(String[] args) {
+		// TODO Auto-generated method stub
 
-    }
+	}
 }

marytts-builder/src/main/java/marytts/tools/voiceimport/vocalizations/MLSAFeatureFileWriter.java

@@ -42,207 +42,216 @@ import marytts.vocalizations.VocalizationUnitFileReader;
 
 /**
  * A component to write all MLSA features (logf0, mgc and strengths) into a single file
+ * 
  * @author sathish
  * 
  */
-public class MLSAFeatureFileWriter extends VoiceImportComponent {    
-    
-    private String lf0Ext  = ".lf0";
-    private String strExt  = ".str";
-    private String mgcExt  = ".mgc";
-    private int progress = 0;
-    protected VocalizationUnitFileReader listenerUnits;
+public class MLSAFeatureFileWriter extends VoiceImportComponent {
 
-    private int MGCORDER = 25; //MGCORDER is actually 24 plus 1 include MGC[0]
-    private int STRORDER = 5;
-    private int LF0ORDER = 1; 
-    
-    protected DatabaseLayout db = null;
-    protected BasenameList bnlVocalizations;
-    
-    public final String MLSADIR = getName()+".vocalizationMLSAFilesDir";
-    public final String UNITFILE = getName()+".unitFile";
-    public final String OUTMLSAFILE = getName()+".mlsaOutputFile";
-    
+	private String lf0Ext = ".lf0";
+	private String strExt = ".str";
+	private String mgcExt = ".mgc";
+	private int progress = 0;
+	protected VocalizationUnitFileReader listenerUnits;
 
-    @Override
-    public SortedMap<String, String> getDefaultProps(DatabaseLayout db) {
-        this.db = db;
-        if (props == null){
-            props = new TreeMap();
-            String mlsaDir = db.getProp(db.VOCALIZATIONSDIR)+"mlsa";
-            props.put(MLSADIR, mlsaDir);
-            props.put(UNITFILE,db.getProp(db.VOCALIZATIONSDIR)+File.separator+"files"+File.separator+"vocalization_units"+db.getProp(db.MARYEXT));
-            props.put(OUTMLSAFILE,db.getProp(db.VOCALIZATIONSDIR)+File.separator+"files"+File.separator+"vocalization_mlsa_features"+db.getProp(db.MARYEXT));
-        }
-        return props;
-    }
-    
-    @Override
-    protected void setupHelp() {
-        
-        if (props2Help ==null){
-            props2Help = new TreeMap();
-            props2Help.put(MLSADIR, "mlsa features directory");
-            props2Help.put(UNITFILE,"unit file representing all vocalizations");
-            props2Help.put(OUTMLSAFILE,"a single file to write all mlsa features");
-        }
-        
-    }
-    
-    @Override
-    public boolean compute() throws Exception {
-        
-        listenerUnits = new VocalizationUnitFileReader(getProp(UNITFILE));
-        
-        // write features into timeline file
-        DataOutputStream out = new DataOutputStream(new BufferedOutputStream(new FileOutputStream(new File(getProp(OUTMLSAFILE)))));
-        writeHeaderTo(out);
-        writeUnitFeaturesTo(out);
-        out.close();
-    
-        MLSAFeatureFileReader tester = new MLSAFeatureFileReader(getProp(OUTMLSAFILE));
-        int unitsOnDisk = tester.getNumberOfUnits();
-        if (unitsOnDisk == listenerUnits.getNumberOfUnits()) {
-            System.out.println("Can read right number of units");
-            return true;
-        } else {
-            System.out.println("Read wrong number of units: "+unitsOnDisk);
-            return false;
-        }
-    }
-    
-    /**
-     * write all features into a single file 
-     * @param out DataOutputStream
-     * @throws IOException if it can't write data into DataOutputStream
-     */
-    private void writeUnitFeaturesTo(DataOutputStream out) throws IOException {
-        
-        int numUnits = listenerUnits.getNumberOfUnits();
-        out.writeInt( numUnits ); // write number of units
-        out.writeInt(this.LF0ORDER); // write lf0 order 
-        out.writeInt(this.MGCORDER); // write MGC order 
-        out.writeInt(this.STRORDER); // write STR order
-        
-        assert bnlVocalizations.getLength() == numUnits : "number of units and size of basename list should be same";
-        
-        for( int i=0; i < bnlVocalizations.getLength(); i++ ) {
-        
-            String mgcFile = getProp(MLSADIR)+File.separator+bnlVocalizations.getName(i)+mgcExt;
-            String lf0File = getProp(MLSADIR)+File.separator+bnlVocalizations.getName(i)+lf0Ext;
-            String strFile = getProp(MLSADIR)+File.separator+bnlVocalizations.getName(i)+strExt;
-            
-            LEDataInputStream lf0Data = new LEDataInputStream (new BufferedInputStream(new FileInputStream(lf0File)));
-            LEDataInputStream mgcData = new LEDataInputStream (new BufferedInputStream(new FileInputStream(mgcFile)));
-            LEDataInputStream strData = new LEDataInputStream (new BufferedInputStream(new FileInputStream(strFile)));
-        
-            int mgcSize = (int) ((new File(mgcFile)).length() / 4); // 4 bytes for float
-            int lf0Size = (int) ((new File(lf0File)).length() / 4); // 4 bytes for float
-            int strSize = (int) ((new File(strFile)).length() / 4); // 4 bytes for float
-            
-            
-            float sizes[] = new float[3];
-            int n=0;
-            sizes[n++] = mgcSize/(this.MGCORDER);
-            sizes[n++] = lf0Size;
-            sizes[n++] = strSize/this.STRORDER;
-            int numberOfFrames = (int) MathUtils.getMin(sizes);
-            out.writeInt(numberOfFrames); // number of frames in this vocalization
-            
-            // first write LF0 data
-            int numberOfLF0Frames = numberOfFrames;
-            out.writeInt(numberOfLF0Frames); // number of LF0 frames 
-            for ( int j=0; j < numberOfLF0Frames; j++ ) {
-                out.writeFloat(lf0Data.readFloat());
-            }
-            
-            // second write MGC data
-            int numberOfMGCFrames = numberOfFrames * (this.MGCORDER);
-            out.writeInt(numberOfMGCFrames); // number of MGC frames 
-            for ( int j=0; j < numberOfMGCFrames; j++ ) {
-                out.writeFloat(mgcData.readFloat());
-            }
-            
-            // Third write STR data
-            int numberOfSTRFrames = numberOfFrames * this.STRORDER;
-            out.writeInt(numberOfSTRFrames); // number of MGC frames 
-            for ( int j=0; j < numberOfSTRFrames; j++ ) {
-                out.writeFloat(strData.readFloat());
-            }
-        }
-    }
+	private int MGCORDER = 25; // MGCORDER is actually 24 plus 1 include MGC[0]
+	private int STRORDER = 5;
+	private int LF0ORDER = 1;
 
-    /**
-     * Initialize this component
-     * @throws MaryConfigurationException if there is problem with basename list
-     */
-    @Override
-    protected void initialiseComp() throws Exception {
-        
-        String timelineDir = db.getProp(db.VOCALIZATIONSDIR) + File.separator + "files";
-        createDirectoryifNotExists(timelineDir);
-        
-        try {
-            String basenameFile = db.getProp(db.VOCALIZATIONSDIR)+File.separator+"basenames.lst";
-            if ( (new File(basenameFile)).exists() ) {
-                System.out.println("Loading basenames of vocalizations from '"+basenameFile+"' list...");
-                bnlVocalizations = new BasenameList(basenameFile);
-                System.out.println("Found "+bnlVocalizations.getLength()+ " vocalizations in basename list");
-            }
-            else {
-                String vocalWavDir = db.getProp(db.VOCALIZATIONSDIR)+File.separator+"wav";
-                System.out.println("Loading basenames of vocalizations from '"+vocalWavDir+"' directory...");
-                bnlVocalizations = new BasenameList(vocalWavDir, ".wav");
-                System.out.println("Found "+bnlVocalizations.getLength()+ " vocalizations in "+ vocalWavDir + " directory");
-            }
-        } catch (IOException e) {
-            throw new MaryConfigurationException("Problem with basename list "+e);
-        }
-    }
-    
-    
-    /**
-     * Write the header of this feature file to the given DataOutput
-     * @param out DataOutput to write header
-     * @throws IOException if it can't write data into DataOutput
-     */
-    protected void writeHeaderTo(DataOutput out) throws IOException {
-        new MaryHeader(MaryHeader.LISTENERFEATS).writeTo(out);
-    }
+	protected DatabaseLayout db = null;
+	protected BasenameList bnlVocalizations;
 
-    
-    /**
-     * Create new directory if the directory doesn't exist 
-     * @param dirName directory path  
-     * @throws Exception if it fails 
-     */
-    private void createDirectoryifNotExists(String dirName) throws Exception {
-        if (!(new File(dirName)).exists()) {
-            System.out.println( dirName+ " directory does not exist; ");
-            if (!(new File(dirName)).mkdirs()) {
-                throw new Exception("Could not create directory "+dirName);
-            }
-            System.out.println("Created successfully.\n");
-        }
-    }
+	public final String MLSADIR = getName() + ".vocalizationMLSAFilesDir";
+	public final String UNITFILE = getName() + ".unitFile";
+	public final String OUTMLSAFILE = getName() + ".mlsaOutputFile";
 
-    @Override
-    public String getName() {
-        return "MLSAFeatureFileWriter";
-    }
+	@Override
+	public SortedMap<String, String> getDefaultProps(DatabaseLayout db) {
+		this.db = db;
+		if (props == null) {
+			props = new TreeMap();
+			String mlsaDir = db.getProp(db.VOCALIZATIONSDIR) + "mlsa";
+			props.put(MLSADIR, mlsaDir);
+			props.put(UNITFILE, db.getProp(db.VOCALIZATIONSDIR) + File.separator + "files" + File.separator
+					+ "vocalization_units" + db.getProp(db.MARYEXT));
+			props.put(OUTMLSAFILE, db.getProp(db.VOCALIZATIONSDIR) + File.separator + "files" + File.separator
+					+ "vocalization_mlsa_features" + db.getProp(db.MARYEXT));
+		}
+		return props;
+	}
 
+	@Override
+	protected void setupHelp() {
 
-    @Override
-    public int getProgress() {
-        return this.progress;
-    }
-    
-    /**
-     * @param args
-     */
-    public static void main(String[] args) {
-        // TODO Auto-generated method stub
+		if (props2Help == null) {
+			props2Help = new TreeMap();
+			props2Help.put(MLSADIR, "mlsa features directory");
+			props2Help.put(UNITFILE, "unit file representing all vocalizations");
+			props2Help.put(OUTMLSAFILE, "a single file to write all mlsa features");
+		}
 
-    }
+	}
+
+	@Override
+	public boolean compute() throws Exception {
+
+		listenerUnits = new VocalizationUnitFileReader(getProp(UNITFILE));
+
+		// write features into timeline file
+		DataOutputStream out = new DataOutputStream(
+				new BufferedOutputStream(new FileOutputStream(new File(getProp(OUTMLSAFILE)))));
+		writeHeaderTo(out);
+		writeUnitFeaturesTo(out);
+		out.close();
+
+		MLSAFeatureFileReader tester = new MLSAFeatureFileReader(getProp(OUTMLSAFILE));
+		int unitsOnDisk = tester.getNumberOfUnits();
+		if (unitsOnDisk == listenerUnits.getNumberOfUnits()) {
+			System.out.println("Can read right number of units");
+			return true;
+		} else {
+			System.out.println("Read wrong number of units: " + unitsOnDisk);
+			return false;
+		}
+	}
+
+	/**
+	 * write all features into a single file
+	 * 
+	 * @param out
+	 *            DataOutputStream
+	 * @throws IOException
+	 *             if it can't write data into DataOutputStream
+	 */
+	private void writeUnitFeaturesTo(DataOutputStream out) throws IOException {
+
+		int numUnits = listenerUnits.getNumberOfUnits();
+		out.writeInt(numUnits); // write number of units
+		out.writeInt(this.LF0ORDER); // write lf0 order
+		out.writeInt(this.MGCORDER); // write MGC order
+		out.writeInt(this.STRORDER); // write STR order
+
+		assert bnlVocalizations.getLength() == numUnits : "number of units and size of basename list should be same";
+
+		for (int i = 0; i < bnlVocalizations.getLength(); i++) {
+
+			String mgcFile = getProp(MLSADIR) + File.separator + bnlVocalizations.getName(i) + mgcExt;
+			String lf0File = getProp(MLSADIR) + File.separator + bnlVocalizations.getName(i) + lf0Ext;
+			String strFile = getProp(MLSADIR) + File.separator + bnlVocalizations.getName(i) + strExt;
+
+			LEDataInputStream lf0Data = new LEDataInputStream(new BufferedInputStream(new FileInputStream(lf0File)));
+			LEDataInputStream mgcData = new LEDataInputStream(new BufferedInputStream(new FileInputStream(mgcFile)));
+			LEDataInputStream strData = new LEDataInputStream(new BufferedInputStream(new FileInputStream(strFile)));
+
+			int mgcSize = (int) ((new File(mgcFile)).length() / 4); // 4 bytes for float
+			int lf0Size = (int) ((new File(lf0File)).length() / 4); // 4 bytes for float
+			int strSize = (int) ((new File(strFile)).length() / 4); // 4 bytes for float
+
+			float sizes[] = new float[3];
+			int n = 0;
+			sizes[n++] = mgcSize / (this.MGCORDER);
+			sizes[n++] = lf0Size;
+			sizes[n++] = strSize / this.STRORDER;
+			int numberOfFrames = (int) MathUtils.getMin(sizes);
+			out.writeInt(numberOfFrames); // number of frames in this vocalization
+
+			// first write LF0 data
+			int numberOfLF0Frames = numberOfFrames;
+			out.writeInt(numberOfLF0Frames); // number of LF0 frames
+			for (int j = 0; j < numberOfLF0Frames; j++) {
+				out.writeFloat(lf0Data.readFloat());
+			}
+
+			// second write MGC data
+			int numberOfMGCFrames = numberOfFrames * (this.MGCORDER);
+			out.writeInt(numberOfMGCFrames); // number of MGC frames
+			for (int j = 0; j < numberOfMGCFrames; j++) {
+				out.writeFloat(mgcData.readFloat());
+			}
+
+			// Third write STR data
+			int numberOfSTRFrames = numberOfFrames * this.STRORDER;
+			out.writeInt(numberOfSTRFrames); // number of MGC frames
+			for (int j = 0; j < numberOfSTRFrames; j++) {
+				out.writeFloat(strData.readFloat());
+			}
+		}
+	}
+
+	/**
+	 * Initialize this component
+	 * 
+	 * @throws MaryConfigurationException
+	 *             if there is problem with basename list
+	 */
+	@Override
+	protected void initialiseComp() throws Exception {
+
+		String timelineDir = db.getProp(db.VOCALIZATIONSDIR) + File.separator + "files";
+		createDirectoryifNotExists(timelineDir);
+
+		try {
+			String basenameFile = db.getProp(db.VOCALIZATIONSDIR) + File.separator + "basenames.lst";
+			if ((new File(basenameFile)).exists()) {
+				System.out.println("Loading basenames of vocalizations from '" + basenameFile + "' list...");
+				bnlVocalizations = new BasenameList(basenameFile);
+				System.out.println("Found " + bnlVocalizations.getLength() + " vocalizations in basename list");
+			} else {
+				String vocalWavDir = db.getProp(db.VOCALIZATIONSDIR) + File.separator + "wav";
+				System.out.println("Loading basenames of vocalizations from '" + vocalWavDir + "' directory...");
+				bnlVocalizations = new BasenameList(vocalWavDir, ".wav");
+				System.out.println("Found " + bnlVocalizations.getLength() + " vocalizations in " + vocalWavDir + " directory");
+			}
+		} catch (IOException e) {
+			throw new MaryConfigurationException("Problem with basename list " + e);
+		}
+	}
+
+	/**
+	 * Write the header of this feature file to the given DataOutput
+	 * 
+	 * @param out
+	 *            DataOutput to write header
+	 * @throws IOException
+	 *             if it can't write data into DataOutput
+	 */
+	protected void writeHeaderTo(DataOutput out) throws IOException {
+		new MaryHeader(MaryHeader.LISTENERFEATS).writeTo(out);
+	}
+
+	/**
+	 * Create new directory if the directory doesn't exist
+	 * 
+	 * @param dirName
+	 *            directory path
+	 * @throws Exception
+	 *             if it fails
+	 */
+	private void createDirectoryifNotExists(String dirName) throws Exception {
+		if (!(new File(dirName)).exists()) {
+			System.out.println(dirName + " directory does not exist; ");
+			if (!(new File(dirName)).mkdirs()) {
+				throw new Exception("Could not create directory " + dirName);
+			}
+			System.out.println("Created successfully.\n");
+		}
+	}
+
+	@Override
+	public String getName() {
+		return "MLSAFeatureFileWriter";
+	}
+
+	@Override
+	public int getProgress() {
+		return this.progress;
+	}
+
+	/**
+	 * @param args
+	 */
+	public static void main(String[] args) {
+		// TODO Auto-generated method stub
+
+	}
 }

marytts-builder/src/main/java/marytts/tools/voiceimport/vocalizations/SnackF0ContourExtractor.java

@@ -41,204 +41,199 @@ import marytts.util.io.General;
 
 public class SnackF0ContourExtractor extends VoiceImportComponent {
 
-    String vocalizationsDir;
-    BasenameList bnlVocalizations;
-    
-    protected DatabaseLayout db = null;
-    //protected String correctedPmExt = ".pm.corrected";
-    protected String snackPmExt = ".snack"; 
-    protected String lf0Ext = ".lf0";
-    protected String scriptFileName;
+	String vocalizationsDir;
+	BasenameList bnlVocalizations;
 
-    private int percent = 0;
-  
-    public final String MINPITCH = "SnackF0ContourExtractor.minPitch";
-    public final String MAXPITCH = "SnackF0ContourExtractor.maxPitch";
-    public final String COMMAND = "SnackF0ContourExtractor.command";
-    public final String LF0DIR = "SnackF0ContourExtractor.pitchContoursDir";
-    public final String WAVEDIR = "SnackF0ContourExtractor.vocalizationWaveDir";
-    public final String SAMPLERATE = "SnackF0ContourExtractor.samplingRate";
-    public final String FRAMEPERIOD = "SnackF0ContourExtractor.framePeriod";
-    
-    public String getName(){
-        return "SnackF0ContourExtractor";
-    }
-    
-    @Override
-    protected void initialiseComp()
-    {
-        scriptFileName = db.getProp(db.VOCALIZATIONSDIR)+"script.snack";
-        if (!(new File(getProp(LF0DIR))).exists()) {
-        
-            System.out.println("vocalizations/lf0 directory does not exist; ");
-            if (!(new File(getProp(LF0DIR))).mkdirs()) {
-                throw new Error("Could not create vocalizations/lf0");
-            }
-            System.out.println("Created successfully.\n");
-            
-        }
-        
-        try {
-            String basenameFile = db.getProp(db.VOCALIZATIONSDIR)+File.separator+"basenames.lst";
-            if ( (new File(basenameFile)).exists() ) {
-                System.out.println("Loading basenames of vocalizations from '"+basenameFile+"' list...");
-                bnlVocalizations = new BasenameList(basenameFile);
-                System.out.println("Found "+bnlVocalizations.getLength()+ " vocalizations in basename list");
-            }
-            else {
-                String vocalWavDir = db.getProp(db.VOCALIZATIONSDIR)+File.separator+"wav";
-                System.out.println("Loading basenames of vocalizations from '"+vocalWavDir+"' directory...");
-                bnlVocalizations = new BasenameList(vocalWavDir, ".wav");
-                System.out.println("Found "+bnlVocalizations.getLength()+ " vocalizations in "+ vocalWavDir + " directory");
-            }
-        } catch (IOException e) {
-            e.printStackTrace();
-        }
-    }
-    
-    public SortedMap getDefaultProps(DatabaseLayout db){
-        this.db = db;
-       if (props == null){
-           props = new TreeMap();
-           props.put(COMMAND,"praat");   
-           if (db.getProp(db.GENDER).equals("female")){
-               props.put(MINPITCH,"100");
-               props.put(MAXPITCH,"500");
-           } else {
-               props.put(MINPITCH,"75");
-               props.put(MAXPITCH,"300");
-           }
-           props.put(WAVEDIR,db.getProp(db.VOCALIZATIONSDIR)+File.separator+"wav");
-           props.put(LF0DIR,db.getProp(db.VOCALIZATIONSDIR)+File.separator+"lf0");
-           props.put(FRAMEPERIOD, "80");
-           props.put(SAMPLERATE, "16000");
-           //vocalizationsDir = db.getProp(db.ROOTDIR)+File.separator+"vocalizations";
-           if (MaryUtils.isWindows())
-               props.put(COMMAND, "c:/tcl/tclsh.exe"); // TODO someone with windows, please confirm or correct
-           else 
-               props.put(COMMAND, "/usr/bin/tclsh");
-       }
-       return props;
-    }
-    
-    /**
-     * The standard compute() method of the VoiceImportComponent interface.
-     * @throws InterruptedException 
-     */
-    public boolean compute() throws IOException, InterruptedException {
-        
-        String[] baseNameArray = bnlVocalizations.getListAsArray();
-        System.out.println( "Computing pitchmarks for " + baseNameArray.length + " utterances." );
-        
-        File script = new File(scriptFileName);
-        
-        // What is the purpose of trunk/marytts/tools/voiceimport/pm.tcl, if it's hardcoded below here?
-        if (script.exists()) script.delete();
-        PrintWriter toScript = new PrintWriter(new FileWriter(script));
-        toScript.println("#!"+getProp(COMMAND));
-        toScript.println(" ");
-        toScript.println("package require snack");
-        toScript.println(" ");
-        toScript.println("snack::sound s");
-        toScript.println(" ");
-        toScript.println("s read [lindex $argv 0]");
-        toScript.println(" ");
-        toScript.println("set frameperiod [ lindex $argv 4 ]");
-        toScript.println("set samplerate [ lindex $argv 5 ]");
-        toScript.println("set framelength [expr {double($frameperiod) / $samplerate}]");
-        toScript.println(" ");
-        toScript.println("set fd [open [lindex $argv 1] w]");
-        toScript.println("puts $fd [join [s pitch -method esps -maxpitch [lindex $argv 2] -minpitch [lindex $argv 3] -framelength $framelength] \\n]");
-        //toScript.println( "puts stdout [ lindex $argv 4 ]");
-        //toScript.println( "puts stdout $framelength");
-        toScript.println("close $fd");
-        toScript.println(" ");
-        toScript.println("exit"); 
-        toScript.println(" ");
-        toScript.close();
+	protected DatabaseLayout db = null;
+	// protected String correctedPmExt = ".pm.corrected";
+	protected String snackPmExt = ".snack";
+	protected String lf0Ext = ".lf0";
+	protected String scriptFileName;
 
-        System.out.println( "Computing pitchmarks for " + baseNameArray.length + " utterances." );
+	private int percent = 0;
 
-        /* Ensure the existence of the target pitchmark directory */
-        /*File dir = new File(db.getProp(PMDIR));
-        if (!dir.exists()) {
-            System.out.println( "Creating the directory [" + db.getProp(PMDIR) + "]." );
-            dir.mkdir();
-        }     */   
-        
-        /* execute snack */        
-        for ( int i = 0; i < baseNameArray.length; i++ ) {
-            percent = 100*i/baseNameArray.length;
-            String wavFile = getProp(WAVEDIR) + baseNameArray[i] + db.getProp(db.WAVEXT);
-            String snackFile = getProp(LF0DIR) + baseNameArray[i] + snackPmExt;
-            String outLF0File = getProp(LF0DIR) + baseNameArray[i] + lf0Ext;
-            System.out.println("Writing pm file to "+snackFile);
+	public final String MINPITCH = "SnackF0ContourExtractor.minPitch";
+	public final String MAXPITCH = "SnackF0ContourExtractor.maxPitch";
+	public final String COMMAND = "SnackF0ContourExtractor.command";
+	public final String LF0DIR = "SnackF0ContourExtractor.pitchContoursDir";
+	public final String WAVEDIR = "SnackF0ContourExtractor.vocalizationWaveDir";
+	public final String SAMPLERATE = "SnackF0ContourExtractor.samplingRate";
+	public final String FRAMEPERIOD = "SnackF0ContourExtractor.framePeriod";
 
-            boolean isWindows = true; // TODO This is WRONG, and never used. Consider removal.
-            String strTmp = scriptFileName + " " + wavFile + " " + snackFile + " " + getProp(MAXPITCH) + " " + getProp(MINPITCH)+" "+getProp(FRAMEPERIOD)+" " + getProp(SAMPLERATE);
+	public String getName() {
+		return "SnackF0ContourExtractor";
+	}
 
-            if (MaryUtils.isWindows())
-                strTmp = "cmd.exe /c " + strTmp;
-            else  strTmp = getProp(COMMAND) + " " + strTmp;
-                
-            General.launchProc(strTmp, "SnackF0ContourExtractor", baseNameArray[i]);
+	@Override
+	protected void initialiseComp() {
+		scriptFileName = db.getProp(db.VOCALIZATIONSDIR) + "script.snack";
+		if (!(new File(getProp(LF0DIR))).exists()) {
 
-            // Now convert the snack format into EST pm format
-            double[] pm = new SnackTextfileDoubleDataSource(new FileReader(snackFile)).getAllData();
-            SPTKPitchReaderWriter sptkPitch = new SPTKPitchReaderWriter(pm, null);
-            sptkPitch.writeIntoSPTKLF0File(outLF0File);
-        }
-        return true;
-        
-    }
-    
-    
-    /**
-     * @param args
-     */
-    public static void main(String[] args) {
-        // TODO Auto-generated method stub
+			System.out.println("vocalizations/lf0 directory does not exist; ");
+			if (!(new File(getProp(LF0DIR))).mkdirs()) {
+				throw new Error("Could not create vocalizations/lf0");
+			}
+			System.out.println("Created successfully.\n");
 
-    }
+		}
 
-    public int getProgress() {
-        return percent;
-    }
+		try {
+			String basenameFile = db.getProp(db.VOCALIZATIONSDIR) + File.separator + "basenames.lst";
+			if ((new File(basenameFile)).exists()) {
+				System.out.println("Loading basenames of vocalizations from '" + basenameFile + "' list...");
+				bnlVocalizations = new BasenameList(basenameFile);
+				System.out.println("Found " + bnlVocalizations.getLength() + " vocalizations in basename list");
+			} else {
+				String vocalWavDir = db.getProp(db.VOCALIZATIONSDIR) + File.separator + "wav";
+				System.out.println("Loading basenames of vocalizations from '" + vocalWavDir + "' directory...");
+				bnlVocalizations = new BasenameList(vocalWavDir, ".wav");
+				System.out.println("Found " + bnlVocalizations.getLength() + " vocalizations in " + vocalWavDir + " directory");
+			}
+		} catch (IOException e) {
+			e.printStackTrace();
+		}
+	}
 
-    protected void setupHelp() {
-        if (props2Help ==null){
-            props2Help = new TreeMap();
-            props2Help.put(COMMAND, "The command that is used to launch snack");
-            props2Help.put(MINPITCH,"minimum value for the pitch (in Hz). Default: female 100, male 75");
-            props2Help.put(MAXPITCH,"maximum value for the pitch (in Hz). Default: female 500, male 300");            
-        }
-    }
-    
-    private static class StreamGobbler extends Thread
-    {
-        InputStream is;
-        String type;
-        
-        public StreamGobbler(InputStream is, String type)
-        {
-            this.is = is;
-            this.type = type;
-        }
-        
-        public void run()
-        {
-            try
-            {
-                InputStreamReader isr = new InputStreamReader(is);
-                BufferedReader br = new BufferedReader(isr);
-                String line=null;
-                while ( (line = br.readLine()) != null)
-                    System.out.println(type + ">" + line);    
-                } catch (IOException ioe)
-                  {
-                    ioe.printStackTrace();  
-                  }
-        }
-    }
+	public SortedMap getDefaultProps(DatabaseLayout db) {
+		this.db = db;
+		if (props == null) {
+			props = new TreeMap();
+			props.put(COMMAND, "praat");
+			if (db.getProp(db.GENDER).equals("female")) {
+				props.put(MINPITCH, "100");
+				props.put(MAXPITCH, "500");
+			} else {
+				props.put(MINPITCH, "75");
+				props.put(MAXPITCH, "300");
+			}
+			props.put(WAVEDIR, db.getProp(db.VOCALIZATIONSDIR) + File.separator + "wav");
+			props.put(LF0DIR, db.getProp(db.VOCALIZATIONSDIR) + File.separator + "lf0");
+			props.put(FRAMEPERIOD, "80");
+			props.put(SAMPLERATE, "16000");
+			// vocalizationsDir = db.getProp(db.ROOTDIR)+File.separator+"vocalizations";
+			if (MaryUtils.isWindows())
+				props.put(COMMAND, "c:/tcl/tclsh.exe"); // TODO someone with windows, please confirm or correct
+			else
+				props.put(COMMAND, "/usr/bin/tclsh");
+		}
+		return props;
+	}
+
+	/**
+	 * The standard compute() method of the VoiceImportComponent interface.
+	 * 
+	 * @throws InterruptedException
+	 */
+	public boolean compute() throws IOException, InterruptedException {
+
+		String[] baseNameArray = bnlVocalizations.getListAsArray();
+		System.out.println("Computing pitchmarks for " + baseNameArray.length + " utterances.");
+
+		File script = new File(scriptFileName);
+
+		// What is the purpose of trunk/marytts/tools/voiceimport/pm.tcl, if it's hardcoded below here?
+		if (script.exists())
+			script.delete();
+		PrintWriter toScript = new PrintWriter(new FileWriter(script));
+		toScript.println("#!" + getProp(COMMAND));
+		toScript.println(" ");
+		toScript.println("package require snack");
+		toScript.println(" ");
+		toScript.println("snack::sound s");
+		toScript.println(" ");
+		toScript.println("s read [lindex $argv 0]");
+		toScript.println(" ");
+		toScript.println("set frameperiod [ lindex $argv 4 ]");
+		toScript.println("set samplerate [ lindex $argv 5 ]");
+		toScript.println("set framelength [expr {double($frameperiod) / $samplerate}]");
+		toScript.println(" ");
+		toScript.println("set fd [open [lindex $argv 1] w]");
+		toScript.println("puts $fd [join [s pitch -method esps -maxpitch [lindex $argv 2] -minpitch [lindex $argv 3] -framelength $framelength] \\n]");
+		// toScript.println( "puts stdout [ lindex $argv 4 ]");
+		// toScript.println( "puts stdout $framelength");
+		toScript.println("close $fd");
+		toScript.println(" ");
+		toScript.println("exit");
+		toScript.println(" ");
+		toScript.close();
+
+		System.out.println("Computing pitchmarks for " + baseNameArray.length + " utterances.");
+
+		/* Ensure the existence of the target pitchmark directory */
+		/*
+		 * File dir = new File(db.getProp(PMDIR)); if (!dir.exists()) { System.out.println( "Creating the directory [" +
+		 * db.getProp(PMDIR) + "]." ); dir.mkdir(); }
+		 */
+
+		/* execute snack */
+		for (int i = 0; i < baseNameArray.length; i++) {
+			percent = 100 * i / baseNameArray.length;
+			String wavFile = getProp(WAVEDIR) + baseNameArray[i] + db.getProp(db.WAVEXT);
+			String snackFile = getProp(LF0DIR) + baseNameArray[i] + snackPmExt;
+			String outLF0File = getProp(LF0DIR) + baseNameArray[i] + lf0Ext;
+			System.out.println("Writing pm file to " + snackFile);
+
+			boolean isWindows = true; // TODO This is WRONG, and never used. Consider removal.
+			String strTmp = scriptFileName + " " + wavFile + " " + snackFile + " " + getProp(MAXPITCH) + " " + getProp(MINPITCH)
+					+ " " + getProp(FRAMEPERIOD) + " " + getProp(SAMPLERATE);
+
+			if (MaryUtils.isWindows())
+				strTmp = "cmd.exe /c " + strTmp;
+			else
+				strTmp = getProp(COMMAND) + " " + strTmp;
+
+			General.launchProc(strTmp, "SnackF0ContourExtractor", baseNameArray[i]);
+
+			// Now convert the snack format into EST pm format
+			double[] pm = new SnackTextfileDoubleDataSource(new FileReader(snackFile)).getAllData();
+			SPTKPitchReaderWriter sptkPitch = new SPTKPitchReaderWriter(pm, null);
+			sptkPitch.writeIntoSPTKLF0File(outLF0File);
+		}
+		return true;
+
+	}
+
+	/**
+	 * @param args
+	 */
+	public static void main(String[] args) {
+		// TODO Auto-generated method stub
+
+	}
+
+	public int getProgress() {
+		return percent;
+	}
+
+	protected void setupHelp() {
+		if (props2Help == null) {
+			props2Help = new TreeMap();
+			props2Help.put(COMMAND, "The command that is used to launch snack");
+			props2Help.put(MINPITCH, "minimum value for the pitch (in Hz). Default: female 100, male 75");
+			props2Help.put(MAXPITCH, "maximum value for the pitch (in Hz). Default: female 500, male 300");
+		}
+	}
+
+	private static class StreamGobbler extends Thread {
+		InputStream is;
+		String type;
+
+		public StreamGobbler(InputStream is, String type) {
+			this.is = is;
+			this.type = type;
+		}
+
+		public void run() {
+			try {
+				InputStreamReader isr = new InputStreamReader(is);
+				BufferedReader br = new BufferedReader(isr);
+				String line = null;
+				while ((line = br.readLine()) != null)
+					System.out.println(type + ">" + line);
+			} catch (IOException ioe) {
+				ioe.printStackTrace();
+			}
+		}
+	}
 
 }

marytts-builder/src/main/java/marytts/tools/voiceimport/vocalizations/VocalizationF0PolyFeatureFileWriter.java

@@ -64,437 +64,421 @@ import marytts.vocalizations.VocalizationUnitFileReader;
 
 /**
  * NOT COMPLETED (USEFUL FOR FUTURE)
+ * 
  * @author sathish
  *
  */
-public class VocalizationF0PolyFeatureFileWriter extends VoiceImportComponent
-{
-    protected File maryDir;
-    protected FeatureFileReader features;
-    protected FeatureDefinition inFeatureDefinition;
-    protected File outFeatureFile;
-    protected FeatureDefinition outFeatureDefinition;
-    protected VocalizationUnitFileReader listenerUnits;
-    protected TimelineReader audio;
-    protected DatabaseLayout db = null;
-    protected int percent = 0;
-    
-    private final String name = "F0PolynomialFeatureFileWriter";
-    public final String UNITFILE = name + ".unitFile";
-    public final String WAVETIMELINE = name + ".waveTimeLine";
-    public final String FEATUREFILE = name + ".featureFile";
-    public final String F0FEATUREFILE = name + ".f0FeatureFile";  
-    public final String POLYNOMORDER = name + ".polynomOrder";
-    public final String SHOWGRAPH = name + ".showGraph";
-    public final String INTERPOLATE = name + ".interpolate";
-    public final String MINPITCH = name + ".minPitch";
-    public final String MAXPITCH = name + ".maxPitch";
-    
-    public String getName(){
-        return name;
-    }
-   
-    
-   public SortedMap<String, String> getDefaultProps(DatabaseLayout db){
-       this.db = db;
-       if (props == null){
-           props = new TreeMap<String, String>();
-           String fileDir = db.getProp(db.FILEDIR);
-           String maryExt = db.getProp(db.MARYEXT);
-           props.put(UNITFILE,fileDir+"halfphoneUnits"+maryExt);
-           props.put(WAVETIMELINE,fileDir+"timeline_waveforms"+maryExt);
-           props.put(FEATUREFILE,fileDir+"halfphoneFeatures"+maryExt);
-           props.put(F0FEATUREFILE,fileDir+"vocalizationF0Polynomials"+maryExt);
-           props.put(POLYNOMORDER, "3");
-           props.put(SHOWGRAPH, "false");
-           props.put(INTERPOLATE, "true");
-           if (db.getProp(db.GENDER).equals("female")){
-               props.put(MINPITCH,"100");
-               props.put(MAXPITCH,"600");
-           } else {
-               props.put(MINPITCH,"60");
-               props.put(MAXPITCH,"400");
-           }
-       }
-       return props;
-   }
-    
-   protected void setupHelp()
-   {
-       if (props2Help ==null) {
-           props2Help = new TreeMap<String, String>();
-           props2Help.put(UNITFILE,"file containing all halfphone units");
-           props2Help.put(WAVETIMELINE, "file containing all waveforms or models that can genarate them"); 
-           props2Help.put(FEATUREFILE,"file containing all halfphone units and their target cost features");
-           props2Help.put(F0FEATUREFILE,"file containing syllable-based polynom coefficients on vowels");
-           props2Help.put(POLYNOMORDER, "order of the polynoms used to approximate syllable F0 curves");
-           props2Help.put(SHOWGRAPH, "whether to show a graph with f0 aproximations for each sentence");
-           props2Help.put(INTERPOLATE, "whether to interpolate F0 across unvoiced regions");
-           props2Help.put(MINPITCH,"minimum value for the pitch (in Hz). Default: female 100, male 75");
-           props2Help.put(MAXPITCH,"maximum value for the pitch (in Hz). Default: female 500, male 300");            
-       }
-   }
-  
-    @Override
-    public boolean compute() throws IOException, MaryConfigurationException
-    {
-        logger.info("F0 polynomial feature file writer started.");
+public class VocalizationF0PolyFeatureFileWriter extends VoiceImportComponent {
+	protected File maryDir;
+	protected FeatureFileReader features;
+	protected FeatureDefinition inFeatureDefinition;
+	protected File outFeatureFile;
+	protected FeatureDefinition outFeatureDefinition;
+	protected VocalizationUnitFileReader listenerUnits;
+	protected TimelineReader audio;
+	protected DatabaseLayout db = null;
+	protected int percent = 0;
 
-        maryDir = new File( db.getProp(db.FILEDIR));
-        if (!maryDir.exists()) {
-            maryDir.mkdirs();
-            System.out.println("Created the output directory [" + ( db.getProp(db.FILEDIR)) + "] to store the feature file." );
-        }
-        listenerUnits = new VocalizationUnitFileReader(getProp(UNITFILE));
-        audio = new TimelineReader(getProp(WAVETIMELINE));
-        
-        //features = new FeatureFileReader(getProp(FEATUREFILE));
-        //inFeatureDefinition = features.getFeatureDefinition();
-        StringWriter sw = new StringWriter();
-        PrintWriter pw = new PrintWriter(sw);
-        pw.println(FeatureDefinition.BYTEFEATURES); // no byte features
-        pw.println(FeatureDefinition.SHORTFEATURES); // no short features
-        pw.println(FeatureDefinition.CONTINUOUSFEATURES);
-        int polynomOrder = Integer.parseInt(getProp(POLYNOMORDER));
-        for (int i=polynomOrder; i >= 0; i--) {
-            pw.println("0 linear | f0contour_a"+i);
-        }
-        pw.close();
-        String fd = sw.toString();
-        logger.debug("Generated the following feature definition:");
-        logger.debug(fd);
-        StringReader sr = new StringReader(fd);
-        BufferedReader br = new BufferedReader(sr);
-        outFeatureDefinition = new FeatureDefinition(br, true);
+	private final String name = "F0PolynomialFeatureFileWriter";
+	public final String UNITFILE = name + ".unitFile";
+	public final String WAVETIMELINE = name + ".waveTimeLine";
+	public final String FEATUREFILE = name + ".featureFile";
+	public final String F0FEATUREFILE = name + ".f0FeatureFile";
+	public final String POLYNOMORDER = name + ".polynomOrder";
+	public final String SHOWGRAPH = name + ".showGraph";
+	public final String INTERPOLATE = name + ".interpolate";
+	public final String MINPITCH = name + ".minPitch";
+	public final String MAXPITCH = name + ".maxPitch";
 
-        outFeatureFile = new File(getProp(F0FEATUREFILE));
-        DataOutputStream out = new DataOutputStream(new BufferedOutputStream(new FileOutputStream(outFeatureFile)));
-        writeHeaderTo(out);
-        writeUnitFeaturesTo(out);
-        out.close();
-        logger.debug("Number of processed units: " + listenerUnits.getNumberOfUnits() );
+	public String getName() {
+		return name;
+	}
 
-        FeatureFileReader tester = FeatureFileReader.getFeatureFileReader(getProp(F0FEATUREFILE));
-        int unitsOnDisk = tester.getNumberOfUnits();
-        if (unitsOnDisk == listenerUnits.getNumberOfUnits()) {
-            System.out.println("Can read right number of units");
-            return true;
-        } else {
-            System.out.println("Read wrong number of units: "+unitsOnDisk);
-            return false;
-        }
-    }
+	public SortedMap<String, String> getDefaultProps(DatabaseLayout db) {
+		this.db = db;
+		if (props == null) {
+			props = new TreeMap<String, String>();
+			String fileDir = db.getProp(db.FILEDIR);
+			String maryExt = db.getProp(db.MARYEXT);
+			props.put(UNITFILE, fileDir + "halfphoneUnits" + maryExt);
+			props.put(WAVETIMELINE, fileDir + "timeline_waveforms" + maryExt);
+			props.put(FEATUREFILE, fileDir + "halfphoneFeatures" + maryExt);
+			props.put(F0FEATUREFILE, fileDir + "vocalizationF0Polynomials" + maryExt);
+			props.put(POLYNOMORDER, "3");
+			props.put(SHOWGRAPH, "false");
+			props.put(INTERPOLATE, "true");
+			if (db.getProp(db.GENDER).equals("female")) {
+				props.put(MINPITCH, "100");
+				props.put(MAXPITCH, "600");
+			} else {
+				props.put(MINPITCH, "60");
+				props.put(MAXPITCH, "400");
+			}
+		}
+		return props;
+	}
 
+	protected void setupHelp() {
+		if (props2Help == null) {
+			props2Help = new TreeMap<String, String>();
+			props2Help.put(UNITFILE, "file containing all halfphone units");
+			props2Help.put(WAVETIMELINE, "file containing all waveforms or models that can genarate them");
+			props2Help.put(FEATUREFILE, "file containing all halfphone units and their target cost features");
+			props2Help.put(F0FEATUREFILE, "file containing syllable-based polynom coefficients on vowels");
+			props2Help.put(POLYNOMORDER, "order of the polynoms used to approximate syllable F0 curves");
+			props2Help.put(SHOWGRAPH, "whether to show a graph with f0 aproximations for each sentence");
+			props2Help.put(INTERPOLATE, "whether to interpolate F0 across unvoiced regions");
+			props2Help.put(MINPITCH, "minimum value for the pitch (in Hz). Default: female 100, male 75");
+			props2Help.put(MAXPITCH, "maximum value for the pitch (in Hz). Default: female 500, male 300");
+		}
+	}
 
+	@Override
+	public boolean compute() throws IOException, MaryConfigurationException {
+		logger.info("F0 polynomial feature file writer started.");
 
-    /**
-     * @param out
-     * @throws IOException
-     * @throws UnsupportedEncodingException
-     * @throws FileNotFoundException
-     */
-    protected void writeUnitFeaturesTo(DataOutput out) throws IOException, UnsupportedEncodingException, FileNotFoundException {
-        int numUnits = listenerUnits.getNumberOfUnits();
-        int unitSampleRate = listenerUnits.getSampleRate();
-        int audioSampleRate = audio.getSampleRate();
-        boolean showGraph = Boolean.parseBoolean(getProp(SHOWGRAPH));
-        boolean interpolate = Boolean.parseBoolean(getProp(INTERPOLATE));
-        int polynomOrder = Integer.parseInt(getProp(POLYNOMORDER));
-        float[] zeros = new float[polynomOrder+1];
-        int unitIndex = 0;
+		maryDir = new File(db.getProp(db.FILEDIR));
+		if (!maryDir.exists()) {
+			maryDir.mkdirs();
+			System.out.println("Created the output directory [" + (db.getProp(db.FILEDIR)) + "] to store the feature file.");
+		}
+		listenerUnits = new VocalizationUnitFileReader(getProp(UNITFILE));
+		audio = new TimelineReader(getProp(WAVETIMELINE));
 
-        out.writeInt( numUnits );
-        logger.debug("Number of units : "+numUnits);
+		// features = new FeatureFileReader(getProp(FEATUREFILE));
+		// inFeatureDefinition = features.getFeatureDefinition();
+		StringWriter sw = new StringWriter();
+		PrintWriter pw = new PrintWriter(sw);
+		pw.println(FeatureDefinition.BYTEFEATURES); // no byte features
+		pw.println(FeatureDefinition.SHORTFEATURES); // no short features
+		pw.println(FeatureDefinition.CONTINUOUSFEATURES);
+		int polynomOrder = Integer.parseInt(getProp(POLYNOMORDER));
+		for (int i = polynomOrder; i >= 0; i--) {
+			pw.println("0 linear | f0contour_a" + i);
+		}
+		pw.close();
+		String fd = sw.toString();
+		logger.debug("Generated the following feature definition:");
+		logger.debug(fd);
+		StringReader sr = new StringReader(fd);
+		BufferedReader br = new BufferedReader(sr);
+		outFeatureDefinition = new FeatureDefinition(br, true);
 
-        FeatureDefinition featureDefinition = features.getFeatureDefinition();
-        int fiPhoneme = featureDefinition.getFeatureIndex("phone");
-        byte fvPhoneme_0 = featureDefinition.getFeatureValueAsByte(fiPhoneme, "0");
-        byte fvPhoneme_Silence = featureDefinition.getFeatureValueAsByte(fiPhoneme, "_");
-        int fiLR = featureDefinition.getFeatureIndex("halfphone_lr");
-        byte fvLR_L = featureDefinition.getFeatureValueAsByte(fiLR, "L");
-        byte fvLR_R = featureDefinition.getFeatureValueAsByte(fiLR, "R");
-        int fiSylStart = featureDefinition.getFeatureIndex("segs_from_syl_start");
-        int fiSylEnd = featureDefinition.getFeatureIndex("segs_from_syl_end");
-        int fiSentenceStart = featureDefinition.getFeatureIndex("words_from_sentence_start");
-        int fiSentenceEnd = featureDefinition.getFeatureIndex("words_from_sentence_end");
-        int fiWordStart = featureDefinition.getFeatureIndex("segs_from_word_start");
-        int fiWordEnd = featureDefinition.getFeatureIndex("segs_from_word_end");
-        int fiVowel = featureDefinition.getFeatureIndex("ph_vc");
-        byte fvVowel_Plus = featureDefinition.getFeatureValueAsByte(fiVowel, "+");
-        
-        boolean haveUnitLogF0 = false;
-        int fiUnitLogF0 = -1;
-        int fiUnitLogF0delta = -1;
-        if (featureDefinition.hasFeature("unit_logf0") && featureDefinition.hasFeature("unit_logf0delta")) {
-            haveUnitLogF0 = true;
-            fiUnitLogF0 = featureDefinition.getFeatureIndex("unit_logf0");
-            fiUnitLogF0delta = featureDefinition.getFeatureIndex("unit_logf0delta");
-        }
-        
-        FunctionGraph f0Graph = null;
-        JFrame jf = null;
-        int iSentenceStart = -1;
-        int iSentenceEnd = -1;
-        List<Integer> iSylStarts = new ArrayList<Integer>(); 
-        List<Integer> iSylEnds = new ArrayList<Integer>();
-        List<Integer> iSylVowels = new ArrayList<Integer>();
-        if (showGraph) {
-            f0Graph = new FunctionGraph(0, 1, new double[1]);
-            f0Graph.setYMinMax(50, 300);
-            f0Graph.setPrimaryDataSeriesStyle(Color.BLUE, FunctionGraph.DRAW_DOTS, FunctionGraph.DOT_FULLCIRCLE);
-            jf = f0Graph.showInJFrame("Sentence", false, true);
-        }
+		outFeatureFile = new File(getProp(F0FEATUREFILE));
+		DataOutputStream out = new DataOutputStream(new BufferedOutputStream(new FileOutputStream(outFeatureFile)));
+		writeHeaderTo(out);
+		writeUnitFeaturesTo(out);
+		out.close();
+		logger.debug("Number of processed units: " + listenerUnits.getNumberOfUnits());
 
-        for (int i=0; i<numUnits; i++) {
-            percent = 100*i/numUnits;
-            FeatureVector fv = features.getFeatureVector(i);
-            //System.out.print(featureDefinition.getFeatureValueAsString("phone", fv));
-            //if (fv.getByteFeature(fiPhoneme) == fvPhoneme_0
-            //        || fv.getByteFeature(fiPhoneme) == fvPhoneme_Silence) continue;
-            if (iSentenceStart == -1
-                    && fv.getByteFeature(fiSentenceStart) == 0
-                    && fv.getByteFeature(fiWordStart) == 0
-                    && fv.getByteFeature(fiLR) == fvLR_L) { // first unit in sentence
-                iSentenceStart = i;
-                iSylStarts.clear();
-                iSylEnds.clear();
-                iSylVowels.clear();
-                //System.out.print(", is sentence start");
-            }
-            // Silence and edge units cannot be part of syllables, but they can 
-            // mark start/end of sentence:
-            if (fv.getByteFeature(fiPhoneme) != fvPhoneme_0
-                    && fv.getByteFeature(fiPhoneme) != fvPhoneme_Silence) {
-                if (fv.getByteFeature(fiSylStart) == 0 && fv.getByteFeature(fiLR) == fvLR_L) { // first segment in syllable
-                    if (iSylStarts.size() > iSylEnds.size()) {
-                        System.err.println("Syllable ends before other syllable starts!");
-                    }
-                    iSylStarts.add(i);
-                    //System.out.print(", is syl start");
-                }
-                if (fv.getByteFeature(fiVowel) == fvVowel_Plus && iSylVowels.size() < iSylStarts.size()) { // first vowel unit in syllable
-                    iSylVowels.add(i);
-                    //System.out.print(", is vowel");
-                }
-                if (fv.getByteFeature(fiSylEnd) == 0 && fv.getByteFeature(fiLR) == fvLR_R) { // last segment in syllable
-                    iSylEnds.add(i);
-                    //System.out.print(", is syl end");
-                    assert iSylStarts.size() == iSylEnds.size();
-                    if (iSylVowels.size() < iSylEnds.size()) {
-                        //System.err.println("Syllable contains no vowel -- skipping");
-                        iSylStarts.remove(iSylStarts.size() - 1);
-                        iSylEnds.remove(iSylEnds.size() - 1);
-                    }
-                }
-            }
-            if (iSentenceStart != -1
-                    && fv.getByteFeature(fiSentenceEnd) == 0
-                    && fv.getByteFeature(fiWordEnd) == 0
-                    && fv.getByteFeature(fiLR) == fvLR_R) { // last unit in sentence
-                iSentenceEnd = i;
-                //System.out.print(", is sentence end");
-                if (iSylEnds.size() < iSylStarts.size()) {
-                    System.err.println("Last syllable in sentence is not properly closed");
-                    iSylEnds.add(i);
-                }
-            }
-            //System.out.println();
-            
-            if (iSentenceStart >= 0 && iSentenceEnd >= iSentenceStart && iSylVowels.size() > 0) {
-                assert iSylStarts.size() == iSylEnds.size() : "Have "+iSylStarts.size()+" syllable starts, but "+iSylEnds.size()+" syllable ends!";
-                assert iSylStarts.size() == iSylVowels.size();
-                long tsSentenceStart = listenerUnits.getUnit(iSentenceStart).startTime;
-                long tsSentenceEnd = listenerUnits.getUnit(iSentenceEnd).startTime + listenerUnits.getUnit(iSentenceEnd).duration;
-                long tsSentenceDuration = tsSentenceEnd - tsSentenceStart;
-                Datagram[] sentenceData = audio.getDatagrams(tsSentenceStart, tsSentenceDuration);
-                DatagramDoubleDataSource ddds = new DatagramDoubleDataSource(sentenceData);
-                double[] sentenceAudio = ddds.getAllData();
-                AudioPlayer ap = null;
-                if (showGraph) {
-                    ap = new AudioPlayer(new DDSAudioInputStream(new BufferedDoubleDataSource(sentenceAudio), new AudioFormat(AudioFormat.Encoding.PCM_SIGNED,
-                            audioSampleRate, // samples per second
-                            16, // bits per sample
-                            1, // mono
-                            2, // nr. of bytes per frame
-                            audioSampleRate, // nr. of frames per second
-                            true))); // big-endian;))
-                    ap.start();
-                }
-                PitchFileHeader params = new PitchFileHeader();
-                params.fs = audioSampleRate;
-                params.minimumF0 = Double.parseDouble(getProp(MINPITCH));
-                params.maximumF0 = Double.parseDouble(getProp(MAXPITCH));
-                F0TrackerAutocorrelationHeuristic tracker = new F0TrackerAutocorrelationHeuristic(params);
-                tracker.pitchAnalyze(new BufferedDoubleDataSource(sentenceAudio));
-                double frameShiftTime = tracker.getSkipSizeInSeconds();
-                double[] f0Array = tracker.getF0Contour();
-                if (f0Array != null) {
-                    for (int j=0; j<f0Array.length; j++) {
-                        if (f0Array[j] == 0) {
-                            f0Array[j] = Double.NaN;
-                        }
-                    }
-                    if (f0Array.length >= 3) {
-                        f0Array = SignalProcUtils.medianFilter(f0Array, 5);
-                    }
-                    if (showGraph) {
-                        f0Graph.updateData(0, tsSentenceDuration / (double) audioSampleRate /f0Array.length, f0Array);
-                        jf.repaint();
-                    }
+		FeatureFileReader tester = FeatureFileReader.getFeatureFileReader(getProp(F0FEATUREFILE));
+		int unitsOnDisk = tester.getNumberOfUnits();
+		if (unitsOnDisk == listenerUnits.getNumberOfUnits()) {
+			System.out.println("Can read right number of units");
+			return true;
+		} else {
+			System.out.println("Read wrong number of units: " + unitsOnDisk);
+			return false;
+		}
+	}
 
-                    double[] f0AndInterpol;
-                    if (interpolate) {
-                        double[] interpol = new double[f0Array.length];
-                        Arrays.fill(interpol, Double.NaN);
-                        f0AndInterpol = new double[f0Array.length];
-                        int iLastValid = -1;
-                        for (int j=0; j<f0Array.length; j++) {
-                            if (!Double.isNaN(f0Array[j])) { // a valid value
-                                if (iLastValid == j-1) {
-                                    // no need to interpolate
-                                    f0AndInterpol[j] = f0Array[j];
-                                } else {
-                                    // need to interpolate
-                                    double prevF0;
-                                    if (iLastValid < 0) { // we don't have a previous value -- use current one
-                                        prevF0 = f0Array[j];
-                                    } else {
-                                        prevF0 = f0Array[iLastValid];
-                                    }
-                                    double delta = (f0Array[j]-prevF0)/(j-iLastValid);
-                                    double f0 = prevF0;
-                                    for (int k = iLastValid+1; k<j; k++) {
-                                        f0 += delta;
-                                        interpol[k] = f0;
-                                        f0AndInterpol[k] = f0;
-                                    }
-                                }
-                                iLastValid = j;
-                            }
-                        }
-                        if (showGraph) {
-                            f0Graph.addDataSeries(interpol, Color.GREEN, FunctionGraph.DRAW_DOTS, FunctionGraph.DOT_EMPTYCIRCLE);
-                            jf.repaint();
-                        }
-                    } else {
-                        f0AndInterpol = f0Array.clone();
-                    }
+	/**
+	 * @param out
+	 * @throws IOException
+	 * @throws UnsupportedEncodingException
+	 * @throws FileNotFoundException
+	 */
+	protected void writeUnitFeaturesTo(DataOutput out) throws IOException, UnsupportedEncodingException, FileNotFoundException {
+		int numUnits = listenerUnits.getNumberOfUnits();
+		int unitSampleRate = listenerUnits.getSampleRate();
+		int audioSampleRate = audio.getSampleRate();
+		boolean showGraph = Boolean.parseBoolean(getProp(SHOWGRAPH));
+		boolean interpolate = Boolean.parseBoolean(getProp(INTERPOLATE));
+		int polynomOrder = Integer.parseInt(getProp(POLYNOMORDER));
+		float[] zeros = new float[polynomOrder + 1];
+		int unitIndex = 0;
 
-                    for (int j=0; j<f0AndInterpol.length; j++) {
-                        if (f0AndInterpol[j] == 0)
-                            f0AndInterpol[j] = Double.NaN;
-                        else
-                            f0AndInterpol[j] = Math.log(f0AndInterpol[j]);
-                    }
-                    double[] approx = new double[f0Array.length];
-                    Arrays.fill(approx, Double.NaN);
-                    for (int s=0; s<iSylStarts.size(); s++) {
-                        long tsSylStart = listenerUnits.getUnit(iSylStarts.get(s)).startTime;
-                        long tsSylEnd = listenerUnits.getUnit(iSylEnds.get(s)).startTime + listenerUnits.getUnit(iSylEnds.get(s)).duration;
-                        long tsSylDuration = tsSylEnd - tsSylStart;
-                        int iSylVowel = iSylVowels.get(s);
-                        // now map time to position in f0AndInterpol array:
-                        int iSylStart = (int) (((double)(tsSylStart-tsSentenceStart)/tsSentenceDuration)*f0AndInterpol.length);
-                        assert iSylStart >= 0;
-                        int iSylEnd = iSylStart + (int) ((double) tsSylDuration / tsSentenceDuration * f0AndInterpol.length) + 1;
-                        if (iSylEnd > f0AndInterpol.length) iSylEnd = f0AndInterpol.length;
-                        //System.out.println("Syl "+s+" from "+iSylStart+" to "+iSylEnd+" out of "+f0AndInterpol.length);
-                        double[] sylF0 = new double[iSylEnd-iSylStart];
-                        System.arraycopy(f0AndInterpol, iSylStart, sylF0, 0, sylF0.length);
-                        double[] coeffs = Polynomial.fitPolynomial(sylF0, polynomOrder);
-                        if (coeffs != null) {
-                            if (showGraph) {
-                                double[] sylPred = Polynomial.generatePolynomialValues(coeffs, sylF0.length, 0, 1);
-                                System.arraycopy(sylPred, 0, approx, iSylStart, sylPred.length);
-                            }
-                            // Write coefficients to file
-                            while (unitIndex < iSylVowel) {
-                                FeatureVector outFV = outFeatureDefinition.toFeatureVector(unitIndex, null, null, zeros);
-                                outFV.writeTo(out);
-                                unitIndex++;
-                            }
-                            float[] fcoeffs = ArrayUtils.copyDouble2Float(coeffs);
-                            //System.out.print("Polynomial values (unit "+unitIndex+") ");
-                            //for (int p=0; p<fcoeffs.length; p++) {
-                            //    System.out.print(", "+fcoeffs[p]);
-                            //}
-                            //System.out.println();
-                            FeatureVector outFV = outFeatureDefinition.toFeatureVector(unitIndex, null, null, fcoeffs);
-                            outFV.writeTo(out);
-                            unitIndex++;
-                        }
-                    }
-                    /*System.out.print("Approximation values that are zero: ");
-                    for (int j=0; j<approx.length; j++) {
-                        if (approx[j] == 0) System.out.print(j+" ");
-                    }
-                    System.out.println();
-                    */
-                    if (showGraph) {
-                        for (int j=0; j<approx.length; j++) {
-                            approx[j] = Math.exp(approx[j]);
-                        }
-                        f0Graph.addDataSeries(approx, Color.RED, FunctionGraph.DRAW_LINE, -1);
-                    }
-                    
-                }
-                if (showGraph) {
-                    try {
-                        ap.join();
-                        Thread.sleep(2000);
-                    } catch (InterruptedException ie) {}
-                }
-                iSentenceStart = -1;
-                iSentenceEnd = -1;
-                iSylStarts.clear();
-                iSylEnds.clear();
-                iSylVowels.clear();
-            }
-        }
-        while (unitIndex < numUnits) {
-            FeatureVector outFV = outFeatureDefinition.toFeatureVector(unitIndex, null, null, zeros);
-            outFV.writeTo(out);
-            unitIndex++;
-        }
+		out.writeInt(numUnits);
+		logger.debug("Number of units : " + numUnits);
 
-    
-    
-    
-    
-    
-    }
+		FeatureDefinition featureDefinition = features.getFeatureDefinition();
+		int fiPhoneme = featureDefinition.getFeatureIndex("phone");
+		byte fvPhoneme_0 = featureDefinition.getFeatureValueAsByte(fiPhoneme, "0");
+		byte fvPhoneme_Silence = featureDefinition.getFeatureValueAsByte(fiPhoneme, "_");
+		int fiLR = featureDefinition.getFeatureIndex("halfphone_lr");
+		byte fvLR_L = featureDefinition.getFeatureValueAsByte(fiLR, "L");
+		byte fvLR_R = featureDefinition.getFeatureValueAsByte(fiLR, "R");
+		int fiSylStart = featureDefinition.getFeatureIndex("segs_from_syl_start");
+		int fiSylEnd = featureDefinition.getFeatureIndex("segs_from_syl_end");
+		int fiSentenceStart = featureDefinition.getFeatureIndex("words_from_sentence_start");
+		int fiSentenceEnd = featureDefinition.getFeatureIndex("words_from_sentence_end");
+		int fiWordStart = featureDefinition.getFeatureIndex("segs_from_word_start");
+		int fiWordEnd = featureDefinition.getFeatureIndex("segs_from_word_end");
+		int fiVowel = featureDefinition.getFeatureIndex("ph_vc");
+		byte fvVowel_Plus = featureDefinition.getFeatureValueAsByte(fiVowel, "+");
 
-    /**
-     * Write the header of this feature file to the given DataOutput
-     * @param out
-     * @throws IOException
-     */
-    protected void writeHeaderTo(DataOutput out) throws IOException {
-        new MaryHeader(MaryHeader.LISTENERFEATS).writeTo(out);
-        outFeatureDefinition.writeBinaryTo(out);
-    }
+		boolean haveUnitLogF0 = false;
+		int fiUnitLogF0 = -1;
+		int fiUnitLogF0delta = -1;
+		if (featureDefinition.hasFeature("unit_logf0") && featureDefinition.hasFeature("unit_logf0delta")) {
+			haveUnitLogF0 = true;
+			fiUnitLogF0 = featureDefinition.getFeatureIndex("unit_logf0");
+			fiUnitLogF0delta = featureDefinition.getFeatureIndex("unit_logf0delta");
+		}
 
-    /**
-     * Provide the progress of computation, in percent, or -1 if
-     * that feature is not implemented.
-     * @return -1 if not implemented, or an integer between 0 and 100.
-     */
-    public int getProgress()
-    {
-        return percent;
-    }
+		FunctionGraph f0Graph = null;
+		JFrame jf = null;
+		int iSentenceStart = -1;
+		int iSentenceEnd = -1;
+		List<Integer> iSylStarts = new ArrayList<Integer>();
+		List<Integer> iSylEnds = new ArrayList<Integer>();
+		List<Integer> iSylVowels = new ArrayList<Integer>();
+		if (showGraph) {
+			f0Graph = new FunctionGraph(0, 1, new double[1]);
+			f0Graph.setYMinMax(50, 300);
+			f0Graph.setPrimaryDataSeriesStyle(Color.BLUE, FunctionGraph.DRAW_DOTS, FunctionGraph.DOT_FULLCIRCLE);
+			jf = f0Graph.showInJFrame("Sentence", false, true);
+		}
 
-    /**
-     * @param args
-     */
-    public static void main(String[] args) throws Exception
-    {
-        VocalizationF0PolyFeatureFileWriter acfeatsWriter = 
-            new VocalizationF0PolyFeatureFileWriter();
-        DatabaseLayout db = new DatabaseLayout(acfeatsWriter);
-        acfeatsWriter.compute();
-    }
+		for (int i = 0; i < numUnits; i++) {
+			percent = 100 * i / numUnits;
+			FeatureVector fv = features.getFeatureVector(i);
+			// System.out.print(featureDefinition.getFeatureValueAsString("phone", fv));
+			// if (fv.getByteFeature(fiPhoneme) == fvPhoneme_0
+			// || fv.getByteFeature(fiPhoneme) == fvPhoneme_Silence) continue;
+			if (iSentenceStart == -1 && fv.getByteFeature(fiSentenceStart) == 0 && fv.getByteFeature(fiWordStart) == 0
+					&& fv.getByteFeature(fiLR) == fvLR_L) { // first unit in sentence
+				iSentenceStart = i;
+				iSylStarts.clear();
+				iSylEnds.clear();
+				iSylVowels.clear();
+				// System.out.print(", is sentence start");
+			}
+			// Silence and edge units cannot be part of syllables, but they can
+			// mark start/end of sentence:
+			if (fv.getByteFeature(fiPhoneme) != fvPhoneme_0 && fv.getByteFeature(fiPhoneme) != fvPhoneme_Silence) {
+				if (fv.getByteFeature(fiSylStart) == 0 && fv.getByteFeature(fiLR) == fvLR_L) { // first segment in syllable
+					if (iSylStarts.size() > iSylEnds.size()) {
+						System.err.println("Syllable ends before other syllable starts!");
+					}
+					iSylStarts.add(i);
+					// System.out.print(", is syl start");
+				}
+				if (fv.getByteFeature(fiVowel) == fvVowel_Plus && iSylVowels.size() < iSylStarts.size()) { // first vowel unit in
+																											// syllable
+					iSylVowels.add(i);
+					// System.out.print(", is vowel");
+				}
+				if (fv.getByteFeature(fiSylEnd) == 0 && fv.getByteFeature(fiLR) == fvLR_R) { // last segment in syllable
+					iSylEnds.add(i);
+					// System.out.print(", is syl end");
+					assert iSylStarts.size() == iSylEnds.size();
+					if (iSylVowels.size() < iSylEnds.size()) {
+						// System.err.println("Syllable contains no vowel -- skipping");
+						iSylStarts.remove(iSylStarts.size() - 1);
+						iSylEnds.remove(iSylEnds.size() - 1);
+					}
+				}
+			}
+			if (iSentenceStart != -1 && fv.getByteFeature(fiSentenceEnd) == 0 && fv.getByteFeature(fiWordEnd) == 0
+					&& fv.getByteFeature(fiLR) == fvLR_R) { // last unit in sentence
+				iSentenceEnd = i;
+				// System.out.print(", is sentence end");
+				if (iSylEnds.size() < iSylStarts.size()) {
+					System.err.println("Last syllable in sentence is not properly closed");
+					iSylEnds.add(i);
+				}
+			}
+			// System.out.println();
 
+			if (iSentenceStart >= 0 && iSentenceEnd >= iSentenceStart && iSylVowels.size() > 0) {
+				assert iSylStarts.size() == iSylEnds.size() : "Have " + iSylStarts.size() + " syllable starts, but "
+						+ iSylEnds.size() + " syllable ends!";
+				assert iSylStarts.size() == iSylVowels.size();
+				long tsSentenceStart = listenerUnits.getUnit(iSentenceStart).startTime;
+				long tsSentenceEnd = listenerUnits.getUnit(iSentenceEnd).startTime + listenerUnits.getUnit(iSentenceEnd).duration;
+				long tsSentenceDuration = tsSentenceEnd - tsSentenceStart;
+				Datagram[] sentenceData = audio.getDatagrams(tsSentenceStart, tsSentenceDuration);
+				DatagramDoubleDataSource ddds = new DatagramDoubleDataSource(sentenceData);
+				double[] sentenceAudio = ddds.getAllData();
+				AudioPlayer ap = null;
+				if (showGraph) {
+					ap = new AudioPlayer(new DDSAudioInputStream(new BufferedDoubleDataSource(sentenceAudio), new AudioFormat(
+							AudioFormat.Encoding.PCM_SIGNED, audioSampleRate, // samples per second
+							16, // bits per sample
+							1, // mono
+							2, // nr. of bytes per frame
+							audioSampleRate, // nr. of frames per second
+							true))); // big-endian;))
+					ap.start();
+				}
+				PitchFileHeader params = new PitchFileHeader();
+				params.fs = audioSampleRate;
+				params.minimumF0 = Double.parseDouble(getProp(MINPITCH));
+				params.maximumF0 = Double.parseDouble(getProp(MAXPITCH));
+				F0TrackerAutocorrelationHeuristic tracker = new F0TrackerAutocorrelationHeuristic(params);
+				tracker.pitchAnalyze(new BufferedDoubleDataSource(sentenceAudio));
+				double frameShiftTime = tracker.getSkipSizeInSeconds();
+				double[] f0Array = tracker.getF0Contour();
+				if (f0Array != null) {
+					for (int j = 0; j < f0Array.length; j++) {
+						if (f0Array[j] == 0) {
+							f0Array[j] = Double.NaN;
+						}
+					}
+					if (f0Array.length >= 3) {
+						f0Array = SignalProcUtils.medianFilter(f0Array, 5);
+					}
+					if (showGraph) {
+						f0Graph.updateData(0, tsSentenceDuration / (double) audioSampleRate / f0Array.length, f0Array);
+						jf.repaint();
+					}
+
+					double[] f0AndInterpol;
+					if (interpolate) {
+						double[] interpol = new double[f0Array.length];
+						Arrays.fill(interpol, Double.NaN);
+						f0AndInterpol = new double[f0Array.length];
+						int iLastValid = -1;
+						for (int j = 0; j < f0Array.length; j++) {
+							if (!Double.isNaN(f0Array[j])) { // a valid value
+								if (iLastValid == j - 1) {
+									// no need to interpolate
+									f0AndInterpol[j] = f0Array[j];
+								} else {
+									// need to interpolate
+									double prevF0;
+									if (iLastValid < 0) { // we don't have a previous value -- use current one
+										prevF0 = f0Array[j];
+									} else {
+										prevF0 = f0Array[iLastValid];
+									}
+									double delta = (f0Array[j] - prevF0) / (j - iLastValid);
+									double f0 = prevF0;
+									for (int k = iLastValid + 1; k < j; k++) {
+										f0 += delta;
+										interpol[k] = f0;
+										f0AndInterpol[k] = f0;
+									}
+								}
+								iLastValid = j;
+							}
+						}
+						if (showGraph) {
+							f0Graph.addDataSeries(interpol, Color.GREEN, FunctionGraph.DRAW_DOTS, FunctionGraph.DOT_EMPTYCIRCLE);
+							jf.repaint();
+						}
+					} else {
+						f0AndInterpol = f0Array.clone();
+					}
+
+					for (int j = 0; j < f0AndInterpol.length; j++) {
+						if (f0AndInterpol[j] == 0)
+							f0AndInterpol[j] = Double.NaN;
+						else
+							f0AndInterpol[j] = Math.log(f0AndInterpol[j]);
+					}
+					double[] approx = new double[f0Array.length];
+					Arrays.fill(approx, Double.NaN);
+					for (int s = 0; s < iSylStarts.size(); s++) {
+						long tsSylStart = listenerUnits.getUnit(iSylStarts.get(s)).startTime;
+						long tsSylEnd = listenerUnits.getUnit(iSylEnds.get(s)).startTime
+								+ listenerUnits.getUnit(iSylEnds.get(s)).duration;
+						long tsSylDuration = tsSylEnd - tsSylStart;
+						int iSylVowel = iSylVowels.get(s);
+						// now map time to position in f0AndInterpol array:
+						int iSylStart = (int) (((double) (tsSylStart - tsSentenceStart) / tsSentenceDuration) * f0AndInterpol.length);
+						assert iSylStart >= 0;
+						int iSylEnd = iSylStart + (int) ((double) tsSylDuration / tsSentenceDuration * f0AndInterpol.length) + 1;
+						if (iSylEnd > f0AndInterpol.length)
+							iSylEnd = f0AndInterpol.length;
+						// System.out.println("Syl "+s+" from "+iSylStart+" to "+iSylEnd+" out of "+f0AndInterpol.length);
+						double[] sylF0 = new double[iSylEnd - iSylStart];
+						System.arraycopy(f0AndInterpol, iSylStart, sylF0, 0, sylF0.length);
+						double[] coeffs = Polynomial.fitPolynomial(sylF0, polynomOrder);
+						if (coeffs != null) {
+							if (showGraph) {
+								double[] sylPred = Polynomial.generatePolynomialValues(coeffs, sylF0.length, 0, 1);
+								System.arraycopy(sylPred, 0, approx, iSylStart, sylPred.length);
+							}
+							// Write coefficients to file
+							while (unitIndex < iSylVowel) {
+								FeatureVector outFV = outFeatureDefinition.toFeatureVector(unitIndex, null, null, zeros);
+								outFV.writeTo(out);
+								unitIndex++;
+							}
+							float[] fcoeffs = ArrayUtils.copyDouble2Float(coeffs);
+							// System.out.print("Polynomial values (unit "+unitIndex+") ");
+							// for (int p=0; p<fcoeffs.length; p++) {
+							// System.out.print(", "+fcoeffs[p]);
+							// }
+							// System.out.println();
+							FeatureVector outFV = outFeatureDefinition.toFeatureVector(unitIndex, null, null, fcoeffs);
+							outFV.writeTo(out);
+							unitIndex++;
+						}
+					}
+					/*
+					 * System.out.print("Approximation values that are zero: "); for (int j=0; j<approx.length; j++) { if
+					 * (approx[j] == 0) System.out.print(j+" "); } System.out.println();
+					 */
+					if (showGraph) {
+						for (int j = 0; j < approx.length; j++) {
+							approx[j] = Math.exp(approx[j]);
+						}
+						f0Graph.addDataSeries(approx, Color.RED, FunctionGraph.DRAW_LINE, -1);
+					}
+
+				}
+				if (showGraph) {
+					try {
+						ap.join();
+						Thread.sleep(2000);
+					} catch (InterruptedException ie) {
+					}
+				}
+				iSentenceStart = -1;
+				iSentenceEnd = -1;
+				iSylStarts.clear();
+				iSylEnds.clear();
+				iSylVowels.clear();
+			}
+		}
+		while (unitIndex < numUnits) {
+			FeatureVector outFV = outFeatureDefinition.toFeatureVector(unitIndex, null, null, zeros);
+			outFV.writeTo(out);
+			unitIndex++;
+		}
+
+	}
+
+	/**
+	 * Write the header of this feature file to the given DataOutput
+	 * 
+	 * @param out
+	 * @throws IOException
+	 */
+	protected void writeHeaderTo(DataOutput out) throws IOException {
+		new MaryHeader(MaryHeader.LISTENERFEATS).writeTo(out);
+		outFeatureDefinition.writeBinaryTo(out);
+	}
+
+	/**
+	 * Provide the progress of computation, in percent, or -1 if that feature is not implemented.
+	 * 
+	 * @return -1 if not implemented, or an integer between 0 and 100.
+	 */
+	public int getProgress() {
+		return percent;
+	}
+
+	/**
+	 * @param args
+	 */
+	public static void main(String[] args) throws Exception {
+		VocalizationF0PolyFeatureFileWriter acfeatsWriter = new VocalizationF0PolyFeatureFileWriter();
+		DatabaseLayout db = new DatabaseLayout(acfeatsWriter);
+		acfeatsWriter.compute();
+	}
 
 }
-

marytts-builder/src/main/java/marytts/tools/voiceimport/vocalizations/VocalizationF0PolynomialInspector.java

@@ -61,418 +61,389 @@ import marytts.util.math.Polynomial;
 import marytts.util.signal.SignalProcUtils;
 import marytts.vocalizations.KMeansClusterer;
 
+public class VocalizationF0PolynomialInspector extends VoiceImportComponent {
+	protected FeatureFileReader features;
+	protected FeatureDefinition inFeatureDefinition;
+	protected UnitFileReader units;
+	protected FeatureFileReader contours;
+	protected TimelineReader audio;
+	protected DatabaseLayout db = null;
+	protected int percent = 0;
+	protected FunctionGraph f0Graph = null;
+	protected JFrame jf = null;
+	protected PrintWriter featurePW;
+	protected double costMeasure = 0;
+	private HashMap<String, Integer> minF0Values;
+	private HashMap<String, Integer> maxF0Values;
+	private Set<String> characters;
 
-public class VocalizationF0PolynomialInspector extends VoiceImportComponent
-{
-    protected FeatureFileReader features;
-    protected FeatureDefinition inFeatureDefinition;
-    protected UnitFileReader units;
-    protected FeatureFileReader contours;
-    protected TimelineReader audio;
-    protected DatabaseLayout db = null;
-    protected int percent = 0;
-    protected FunctionGraph f0Graph = null;
-    protected JFrame jf = null;
-    protected PrintWriter featurePW;
-    protected double costMeasure = 0;
-    private HashMap<String, Integer> minF0Values;
-    private HashMap<String, Integer> maxF0Values;
-    private Set<String> characters;
-    
-    protected BasenameList bnlVocalizations;
-    
-    private final String name = "VocalizationF0PolynomialInspector";
-    public final String WAVEDIR = name + ".waveDir";
-    public final String F0POLYFILE = name + ".f0PolynomialFeatureFile";
-    public final String F0MIN = name + ".f0Minimum";
-    public final String F0MAX = name + ".f0Maximum";
-    public final String PARTBASENAME = name + ".partBaseName";
-    public final String ONEWORD = name + ".oneWordDescription";
-    public final String KCLUSTERS = name + ".numberOfClusters";
-    public final String POLYORDER = name + ".polynomialOrder";
-    public final String ISEXTERNALF0 = name + ".isExternalF0Usage";
-    public final String EXTERNALF0FORMAT = name + ".externalF0Format";
-    public final String EXTERNALDIR = name + ".externalF0Directory";
-    public final String EXTERNALEXT = name + ".externalF0Extention";
-    
-    public String getName(){
-        return name;
-    }
-   
-    
-   public SortedMap<String, String> getDefaultProps(DatabaseLayout db){
-       this.db = db;
-       if (props == null){
-           props = new TreeMap<String, String>();
-           props.put(WAVEDIR,db.getProp(db.VOCALIZATIONSDIR)+File.separator+"wav");
-           props.put(F0POLYFILE,"VocalizationF0PolyFeatureFile.txt");
-           props.put(PARTBASENAME,"");
-           props.put(ONEWORD,"");
-           props.put(F0MIN,"50");
-           props.put(F0MAX,"500");
-           props.put(KCLUSTERS,"15");
-           props.put(POLYORDER,"3");
-           props.put(ISEXTERNALF0,"true");
-           props.put(EXTERNALF0FORMAT,"sptk");
-           props.put(EXTERNALDIR, "lf0");
-           props.put(EXTERNALEXT, ".lf0");
-       }
-       return props;
-   }
-       
-   protected void setupHelp()
-   {
-       if (props2Help ==null) {
-           props2Help = new TreeMap<String, String>();
-           props2Help.put(WAVEDIR, "dir containing all waveforms "); 
-       }
-   }
-  
-   
-   @Override
-   protected void initialiseComp()
-   {
-       minF0Values = new HashMap<String, Integer>();
-       maxF0Values = new HashMap<String, Integer>();
-       minF0Values.put("Spike", 50);
-       minF0Values.put("Poppy", 170);
-       minF0Values.put("Obadiah", 70);
-       minF0Values.put("Prudence", 130);
-       maxF0Values.put("Spike", 150);
-       maxF0Values.put("Poppy", 380);
-       maxF0Values.put("Obadiah", 150);
-       maxF0Values.put("Prudence", 280);
-       
-       characters = new HashSet<String>();
-       characters.add("Spike");
-       characters.add("Poppy");
-       characters.add("Obadiah");
-       characters.add("Prudence");
-       
-       try {
-           String basenameFile = db.getProp(db.VOCALIZATIONSDIR)+File.separator+"basenames.lst";
-           if ( (new File(basenameFile)).exists() ) {
-               System.out.println("Loading basenames of vocalizations from '"+basenameFile+"' list...");
-               bnlVocalizations = new BasenameList(basenameFile);
-               System.out.println("Found "+bnlVocalizations.getLength()+ " vocalizations in basename list");
-           }
-           else {
-               String vocalWavDir = db.getProp(db.VOCALIZATIONSDIR)+File.separator+"wav";
-               System.out.println("Loading basenames of vocalizations from '"+vocalWavDir+"' directory...");
-               bnlVocalizations = new BasenameList(vocalWavDir, ".wav");
-               System.out.println("Found "+bnlVocalizations.getLength()+ " vocalizations in "+ vocalWavDir + " directory");
-           }
-       } catch (IOException e) {
-           e.printStackTrace();
-       }
-   }
-   
-    public boolean compute() throws IOException, UnsupportedAudioFileException
-    {
-        logger.info("F0 polynomial feature file writer started.");
-        f0Graph = new FunctionGraph(0, 1, new double[1]);
-        f0Graph.setYMinMax(50, 550);
-        f0Graph.setPrimaryDataSeriesStyle(Color.BLUE, FunctionGraph.DRAW_DOTS, FunctionGraph.DOT_FULLCIRCLE);
-        jf = f0Graph.showInJFrame("Sentence", false, true);
-        
-        String outPutFile = db.getProp(db.ROOTDIR)+File.separator+getProp(ONEWORD)+getProp(PARTBASENAME)+getProp(F0POLYFILE); 
-        featurePW = new PrintWriter(new FileWriter(new File(outPutFile)));
-        
-        for (int i=0; i<bnlVocalizations.getLength(); i++) {
-            percent = 100*i/bnlVocalizations.getLength();
-            displaySentences(bnlVocalizations.getName(i));
-            
-        }
-        featurePW.flush();
-        featurePW.close();
-        System.out.println("Total Cost : "+costMeasure / (double) bnlVocalizations.getLength());
-        
-        
-        //String fileName = "/home/sathish/phd/voices/en-GB-listener/vocal-polynomials/SpiVocalizationF0PolyFeatureFile.txt";
-        int kValue = (new Integer(getProp(KCLUSTERS))).intValue();
-        KMeansClusterer kmc = new KMeansClusterer();
-        kmc.loadF0Polynomials(outPutFile);
-        kmc.trainer(kValue);
-        //System.exit(0);
-        
-        return true;
-    }
+	protected BasenameList bnlVocalizations;
 
+	private final String name = "VocalizationF0PolynomialInspector";
+	public final String WAVEDIR = name + ".waveDir";
+	public final String F0POLYFILE = name + ".f0PolynomialFeatureFile";
+	public final String F0MIN = name + ".f0Minimum";
+	public final String F0MAX = name + ".f0Maximum";
+	public final String PARTBASENAME = name + ".partBaseName";
+	public final String ONEWORD = name + ".oneWordDescription";
+	public final String KCLUSTERS = name + ".numberOfClusters";
+	public final String POLYORDER = name + ".polynomialOrder";
+	public final String ISEXTERNALF0 = name + ".isExternalF0Usage";
+	public final String EXTERNALF0FORMAT = name + ".externalF0Format";
+	public final String EXTERNALDIR = name + ".externalF0Directory";
+	public final String EXTERNALEXT = name + ".externalF0Extention";
 
+	public String getName() {
+		return name;
+	}
 
-    /**
-     * @param out
-     * @throws IOException
-     * @throws UnsupportedAudioFileException 
-     * @throws UnsupportedEncodingException
-     * @throws FileNotFoundException
-     */
-    protected void displaySentences(String baseName) throws IOException, UnsupportedAudioFileException
-    {
-        /*int numUnits = units.getNumberOfUnits();
-        int unitSampleRate = units.getSampleRate();
-        int audioSampleRate = audio.getSampleRate();
-        int unitIndex = 0;
+	public SortedMap<String, String> getDefaultProps(DatabaseLayout db) {
+		this.db = db;
+		if (props == null) {
+			props = new TreeMap<String, String>();
+			props.put(WAVEDIR, db.getProp(db.VOCALIZATIONSDIR) + File.separator + "wav");
+			props.put(F0POLYFILE, "VocalizationF0PolyFeatureFile.txt");
+			props.put(PARTBASENAME, "");
+			props.put(ONEWORD, "");
+			props.put(F0MIN, "50");
+			props.put(F0MAX, "500");
+			props.put(KCLUSTERS, "15");
+			props.put(POLYORDER, "3");
+			props.put(ISEXTERNALF0, "true");
+			props.put(EXTERNALF0FORMAT, "sptk");
+			props.put(EXTERNALDIR, "lf0");
+			props.put(EXTERNALEXT, ".lf0");
+		}
+		return props;
+	}
 
-        logger.debug("Number of units : "+numUnits);*/
-        String partBaseName = getProp(PARTBASENAME).trim();
-        if ( !"".equals(partBaseName) && !baseName.contains(partBaseName) ){
-            return;
-        }
-        
-        /*
-        String targetDescription = getProp(ONEWORD).trim();
-        String textFileName = db.getProp(db.TEXTDIR)+File.separator+baseName+db.getProp(db.TEXTEXT);
-        String audioDescriprion  = FileUtils.getFileAsString(new File(textFileName), "UTF-8"); 
-        
-        if ( !"".equals(targetDescription) && !targetDescription.equals(audioDescriprion.trim())) {
-            return;
-        }*/
-        
-        //if ( !baseName.equals("") && !baseName.contains(getProp(PARTBASENAME)) ){
-        //    return;
-        //}
+	protected void setupHelp() {
+		if (props2Help == null) {
+			props2Help = new TreeMap<String, String>();
+			props2Help.put(WAVEDIR, "dir containing all waveforms ");
+		}
+	}
 
-        String waveFile = getProp(WAVEDIR)+File.separator+baseName+db.getProp(db.WAVEXT);
-        AudioInputStream inputAudio = AudioSystem.getAudioInputStream(new File(waveFile));
-        
-        // Enforce PCM_SIGNED encoding
-        if (!inputAudio.getFormat().getEncoding().equals(AudioFormat.Encoding.PCM_SIGNED)) {
-            inputAudio = AudioSystem.getAudioInputStream(AudioFormat.Encoding.PCM_SIGNED, inputAudio);
-        }
-        
-        int audioSampleRate = (int)inputAudio.getFormat().getSampleRate();
-        AudioDoubleDataSource signal = new AudioDoubleDataSource(inputAudio);
-        double [] sentenceAudio = signal.getAllData(); //Copies all samples in wav file into a double buffer 
-        long tsSentenceDuration = sentenceAudio.length;
-        
-        
-        
+	@Override
+	protected void initialiseComp() {
+		minF0Values = new HashMap<String, Integer>();
+		maxF0Values = new HashMap<String, Integer>();
+		minF0Values.put("Spike", 50);
+		minF0Values.put("Poppy", 170);
+		minF0Values.put("Obadiah", 70);
+		minF0Values.put("Prudence", 130);
+		maxF0Values.put("Spike", 150);
+		maxF0Values.put("Poppy", 380);
+		maxF0Values.put("Obadiah", 150);
+		maxF0Values.put("Prudence", 280);
 
-        /*Datagram[] sentenceData = audio.getDatagrams(tsSentenceStart, tsSentenceDuration);
-        DatagramDoubleDataSource ddds = new DatagramDoubleDataSource(sentenceData);
-        double[] sentenceAudio = ddds.getAllData();*/
-        
-        AudioPlayer ap = null;
-        ap = new AudioPlayer(new DDSAudioInputStream(new BufferedDoubleDataSource(sentenceAudio), new AudioFormat(AudioFormat.Encoding.PCM_SIGNED,
-                audioSampleRate, // samples per second
-                16, // bits per sample
-                1, // mono
-                2, // nr. of bytes per frame
-                audioSampleRate, // nr. of frames per second
-                true))); // big-endian;))
-        ap.start();
-        
-        PitchFileHeader params = new PitchFileHeader();
-        //params.minimumF0 = 50;
-        //params.maximumF0 = 250;
-        //params.minimumF0 = (new Double(getProp(F0MIN))).doubleValue();
-        //params.maximumF0 = (new Double(getProp(F0MAX))).doubleValue();
-        String character = getCharacterName(baseName);
-        params.minimumF0 = (minF0Values.get(character)).doubleValue();
-        params.maximumF0 = (maxF0Values.get(character)).doubleValue();
-        params.fs = audioSampleRate;
-        
-        double[] f0Array = null;
-        
-        if ( "true".equals(getProp(ISEXTERNALF0)) ) {
-            
-            String externalFormat = getProp(EXTERNALF0FORMAT);
-            String externalDir    = getProp(EXTERNALDIR);
-            String externalExt    = getProp(EXTERNALEXT);
-            
-            if ( "sptk".equals(externalFormat) ) {
-                String fileName = db.getProp(db.VOCALIZATIONSDIR) + File.separator + externalDir + File.separator + baseName + externalExt;
-                SPTKPitchReaderWriter sprw = new SPTKPitchReaderWriter(fileName);
-                f0Array = sprw.getF0Contour();
-            }
-            else if ( "ptc".equals(externalFormat) ) {
-                String fileName = db.getProp(db.ROOTDIR) + File.separator + externalDir + File.separator + baseName + externalExt;
-                PitchReaderWriter sprw = new PitchReaderWriter(fileName);
-                f0Array = sprw.contour;
-            }
-        }
-        else {
-            F0TrackerAutocorrelationHeuristic tracker = new F0TrackerAutocorrelationHeuristic(params);
-            tracker.pitchAnalyze(new BufferedDoubleDataSource(sentenceAudio));
-            //double frameShiftTime = tracker.getSkipSizeInSeconds();
-            f0Array = tracker.getF0Contour();
-        }
-                
-        //f0Array = cutStartEndUnvoicedSegments(f0Array);
-        
-        if (f0Array != null) {
-            for (int j=0; j<f0Array.length; j++) {
-                if (f0Array[j] == 0) {
-                    f0Array[j] = Double.NaN;
-                }
-            }
-            if (f0Array.length >= 3) {
-                f0Array = SignalProcUtils.medianFilter(f0Array, 5);
-            }
-            f0Graph.updateData(0, tsSentenceDuration / (double) audioSampleRate /f0Array.length, f0Array);
-            jf.repaint();
+		characters = new HashSet<String>();
+		characters.add("Spike");
+		characters.add("Poppy");
+		characters.add("Obadiah");
+		characters.add("Prudence");
 
-            double[] f0AndInterpol;
-            double[] interpol = new double[f0Array.length];
-            Arrays.fill(interpol, Double.NaN);
-            f0AndInterpol = new double[f0Array.length];
-            int iLastValid = -1;
-            for (int j=0; j<f0Array.length; j++) {
-                if (!Double.isNaN(f0Array[j])) { // a valid value
-                    if (iLastValid == j-1) {
-                        // no need to interpolate
-                        f0AndInterpol[j] = f0Array[j];
-                    } else {
-                        // need to interpolate
-                        double prevF0;
-                        if (iLastValid < 0) { // we don't have a previous value -- use current one
-                            prevF0 = f0Array[j];
-                        } else {
-                            prevF0 = f0Array[iLastValid];
-                        }
-                        double delta = (f0Array[j]-prevF0)/(j-iLastValid);
-                        double f0 = prevF0;
-                        for (int k = iLastValid+1; k<j; k++) {
-                            f0 += delta;
-                            interpol[k] = f0;
-                            f0AndInterpol[k] = f0;
-                        }
-                    }
-                    iLastValid = j;
-                }
-            }
-            f0Graph.addDataSeries(interpol, Color.GREEN, FunctionGraph.DRAW_DOTS, FunctionGraph.DOT_EMPTYCIRCLE);
-            jf.repaint();
+		try {
+			String basenameFile = db.getProp(db.VOCALIZATIONSDIR) + File.separator + "basenames.lst";
+			if ((new File(basenameFile)).exists()) {
+				System.out.println("Loading basenames of vocalizations from '" + basenameFile + "' list...");
+				bnlVocalizations = new BasenameList(basenameFile);
+				System.out.println("Found " + bnlVocalizations.getLength() + " vocalizations in basename list");
+			} else {
+				String vocalWavDir = db.getProp(db.VOCALIZATIONSDIR) + File.separator + "wav";
+				System.out.println("Loading basenames of vocalizations from '" + vocalWavDir + "' directory...");
+				bnlVocalizations = new BasenameList(vocalWavDir, ".wav");
+				System.out.println("Found " + bnlVocalizations.getLength() + " vocalizations in " + vocalWavDir + " directory");
+			}
+		} catch (IOException e) {
+			e.printStackTrace();
+		}
+	}
 
-            double[] f0AndInterpolate = combineF0andInterpolate(f0Array, interpol);
-            
-            //double[] coeffs = Polynomial.fitPolynomial(sylF0, polynomOrder);
-            int polynomialOrder = (new Integer(getProp(POLYORDER))).intValue();
-            double[] coeffs = Polynomial.fitPolynomial(f0AndInterpolate, polynomialOrder);
-            
-            if (coeffs != null) {
-                double[] sylPred = Polynomial.generatePolynomialValues(coeffs, interpol.length, 0, 1);
-                f0Graph.addDataSeries(sylPred, Color.RED, FunctionGraph.DRAW_LINE, -1);
-                double eqDistance = DistanceComputer.getEuclideanDistance(sylPred, f0AndInterpolate);
-                System.out.println(baseName+" - EqDist: "+ eqDistance / (double) sylPred.length);
-                costMeasure += (eqDistance / (double) sylPred.length);
-                featurePW.print(baseName+" ");
-                for(double c : coeffs){
-                    featurePW.print(c+" ");
-                }
-                featurePW.println();
-                
-            }
-            
-            
-            //double[] pred = Polynomial.generatePolynomialValues(coeffs, iUnitDurationInArray, 0, 1);
-            //f0Graph.addDataSeries(unitF0, Color.BLACK, FunctionGraph.DRAW_LINE, -1);
-        
-            
-            try {
-                ap.join();
-                Thread.sleep(10);
-            } catch (InterruptedException ie) {}
-            //System.out.println();
-            
-      
-        }
-    
-    }
+	public boolean compute() throws IOException, UnsupportedAudioFileException {
+		logger.info("F0 polynomial feature file writer started.");
+		f0Graph = new FunctionGraph(0, 1, new double[1]);
+		f0Graph.setYMinMax(50, 550);
+		f0Graph.setPrimaryDataSeriesStyle(Color.BLUE, FunctionGraph.DRAW_DOTS, FunctionGraph.DOT_FULLCIRCLE);
+		jf = f0Graph.showInJFrame("Sentence", false, true);
 
+		String outPutFile = db.getProp(db.ROOTDIR) + File.separator + getProp(ONEWORD) + getProp(PARTBASENAME)
+				+ getProp(F0POLYFILE);
+		featurePW = new PrintWriter(new FileWriter(new File(outPutFile)));
 
-    private double[] cutStartEndUnvoicedSegments(double[] array) {
-        
-        if(array == null) return null;
-            
-        int startIndex = 0;
-        int endIndex   = array.length;
-        
-        // find start index
-        for ( int i=0; i<array.length; i++ ) {
-            if (array[i] != 0){
-                startIndex = i;
-                break;
-            }
-        }
-        
-        // find end index
-        for ( int i=(array.length - 1); i>0 ; i-- ) {
-            if (array[i] != 0){
-                endIndex = i;
-                break;
-            }
-        }
-        
-        int newArraySize = endIndex - startIndex;
-        
-        
-        double[] newArray = new double[newArraySize];
-        System.arraycopy(array, startIndex, newArray, 0, newArraySize);
-        
-        for ( int i=0; i<newArray.length; i++ ) {
-            System.out.println(newArray[i]);
-        }
-        System.out.println("Resized from "+array.length+" to "+newArraySize);
-        
-        return newArray;
-    }
+		for (int i = 0; i < bnlVocalizations.getLength(); i++) {
+			percent = 100 * i / bnlVocalizations.getLength();
+			displaySentences(bnlVocalizations.getName(i));
 
+		}
+		featurePW.flush();
+		featurePW.close();
+		System.out.println("Total Cost : " + costMeasure / (double) bnlVocalizations.getLength());
 
-    private String getCharacterName(String baseName) {
-        Iterator<String> it = characters.iterator();
-        while(it.hasNext()){
-            String character = it.next().trim();
-            if ( baseName.startsWith(character) ){
-                return character;
-            }
-        }
-        return null;
-    }
+		// String fileName = "/home/sathish/phd/voices/en-GB-listener/vocal-polynomials/SpiVocalizationF0PolyFeatureFile.txt";
+		int kValue = (new Integer(getProp(KCLUSTERS))).intValue();
+		KMeansClusterer kmc = new KMeansClusterer();
+		kmc.loadF0Polynomials(outPutFile);
+		kmc.trainer(kValue);
+		// System.exit(0);
 
+		return true;
+	}
 
-    private double[] combineF0andInterpolate(double[] f0Array, double[] interpol) {
-        
-        double[] f0AndInterpolate = new double[f0Array.length];
-        Arrays.fill(f0AndInterpolate, Double.NaN);
-        for(int i=0; i<f0Array.length; i++) {
-            if( !Double.isNaN(f0Array[i]) ) {
-                f0AndInterpolate[i] = f0Array[i];
-            }
-            else if ( !Double.isNaN(interpol[i]) ) {
-                f0AndInterpolate[i] = interpol[i];
-            }
-            //System.out.println(f0Array[i]+" "+interpol[i]+" "+f0AndInterpolate[i]);
-        }
-        
-        
-        
-        return f0AndInterpolate;
-    }
+	/**
+	 * @param out
+	 * @throws IOException
+	 * @throws UnsupportedAudioFileException
+	 * @throws UnsupportedEncodingException
+	 * @throws FileNotFoundException
+	 */
+	protected void displaySentences(String baseName) throws IOException, UnsupportedAudioFileException {
+		/*
+		 * int numUnits = units.getNumberOfUnits(); int unitSampleRate = units.getSampleRate(); int audioSampleRate =
+		 * audio.getSampleRate(); int unitIndex = 0;
+		 * 
+		 * logger.debug("Number of units : "+numUnits);
+		 */
+		String partBaseName = getProp(PARTBASENAME).trim();
+		if (!"".equals(partBaseName) && !baseName.contains(partBaseName)) {
+			return;
+		}
 
+		/*
+		 * String targetDescription = getProp(ONEWORD).trim(); String textFileName =
+		 * db.getProp(db.TEXTDIR)+File.separator+baseName+db.getProp(db.TEXTEXT); String audioDescriprion =
+		 * FileUtils.getFileAsString(new File(textFileName), "UTF-8");
+		 * 
+		 * if ( !"".equals(targetDescription) && !targetDescription.equals(audioDescriprion.trim())) { return; }
+		 */
 
-    /**
-     * Provide the progress of computation, in percent, or -1 if
-     * that feature is not implemented.
-     * @return -1 if not implemented, or an integer between 0 and 100.
-     */
-    public int getProgress()
-    {
-        return percent;
-    }
+		// if ( !baseName.equals("") && !baseName.contains(getProp(PARTBASENAME)) ){
+		// return;
+		// }
 
-    /**
-     * @param args
-     */
-    public static void main(String[] args) throws Exception
-    {
-        VocalizationF0PolynomialInspector acfeatsWriter = 
-            new VocalizationF0PolynomialInspector();
-        DatabaseLayout db = new DatabaseLayout(acfeatsWriter);
-        acfeatsWriter.compute();
-    }
+		String waveFile = getProp(WAVEDIR) + File.separator + baseName + db.getProp(db.WAVEXT);
+		AudioInputStream inputAudio = AudioSystem.getAudioInputStream(new File(waveFile));
 
+		// Enforce PCM_SIGNED encoding
+		if (!inputAudio.getFormat().getEncoding().equals(AudioFormat.Encoding.PCM_SIGNED)) {
+			inputAudio = AudioSystem.getAudioInputStream(AudioFormat.Encoding.PCM_SIGNED, inputAudio);
+		}
+
+		int audioSampleRate = (int) inputAudio.getFormat().getSampleRate();
+		AudioDoubleDataSource signal = new AudioDoubleDataSource(inputAudio);
+		double[] sentenceAudio = signal.getAllData(); // Copies all samples in wav file into a double buffer
+		long tsSentenceDuration = sentenceAudio.length;
+
+		/*
+		 * Datagram[] sentenceData = audio.getDatagrams(tsSentenceStart, tsSentenceDuration); DatagramDoubleDataSource ddds = new
+		 * DatagramDoubleDataSource(sentenceData); double[] sentenceAudio = ddds.getAllData();
+		 */
+
+		AudioPlayer ap = null;
+		ap = new AudioPlayer(new DDSAudioInputStream(new BufferedDoubleDataSource(sentenceAudio), new AudioFormat(
+				AudioFormat.Encoding.PCM_SIGNED, audioSampleRate, // samples per second
+				16, // bits per sample
+				1, // mono
+				2, // nr. of bytes per frame
+				audioSampleRate, // nr. of frames per second
+				true))); // big-endian;))
+		ap.start();
+
+		PitchFileHeader params = new PitchFileHeader();
+		// params.minimumF0 = 50;
+		// params.maximumF0 = 250;
+		// params.minimumF0 = (new Double(getProp(F0MIN))).doubleValue();
+		// params.maximumF0 = (new Double(getProp(F0MAX))).doubleValue();
+		String character = getCharacterName(baseName);
+		params.minimumF0 = (minF0Values.get(character)).doubleValue();
+		params.maximumF0 = (maxF0Values.get(character)).doubleValue();
+		params.fs = audioSampleRate;
+
+		double[] f0Array = null;
+
+		if ("true".equals(getProp(ISEXTERNALF0))) {
+
+			String externalFormat = getProp(EXTERNALF0FORMAT);
+			String externalDir = getProp(EXTERNALDIR);
+			String externalExt = getProp(EXTERNALEXT);
+
+			if ("sptk".equals(externalFormat)) {
+				String fileName = db.getProp(db.VOCALIZATIONSDIR) + File.separator + externalDir + File.separator + baseName
+						+ externalExt;
+				SPTKPitchReaderWriter sprw = new SPTKPitchReaderWriter(fileName);
+				f0Array = sprw.getF0Contour();
+			} else if ("ptc".equals(externalFormat)) {
+				String fileName = db.getProp(db.ROOTDIR) + File.separator + externalDir + File.separator + baseName + externalExt;
+				PitchReaderWriter sprw = new PitchReaderWriter(fileName);
+				f0Array = sprw.contour;
+			}
+		} else {
+			F0TrackerAutocorrelationHeuristic tracker = new F0TrackerAutocorrelationHeuristic(params);
+			tracker.pitchAnalyze(new BufferedDoubleDataSource(sentenceAudio));
+			// double frameShiftTime = tracker.getSkipSizeInSeconds();
+			f0Array = tracker.getF0Contour();
+		}
+
+		// f0Array = cutStartEndUnvoicedSegments(f0Array);
+
+		if (f0Array != null) {
+			for (int j = 0; j < f0Array.length; j++) {
+				if (f0Array[j] == 0) {
+					f0Array[j] = Double.NaN;
+				}
+			}
+			if (f0Array.length >= 3) {
+				f0Array = SignalProcUtils.medianFilter(f0Array, 5);
+			}
+			f0Graph.updateData(0, tsSentenceDuration / (double) audioSampleRate / f0Array.length, f0Array);
+			jf.repaint();
+
+			double[] f0AndInterpol;
+			double[] interpol = new double[f0Array.length];
+			Arrays.fill(interpol, Double.NaN);
+			f0AndInterpol = new double[f0Array.length];
+			int iLastValid = -1;
+			for (int j = 0; j < f0Array.length; j++) {
+				if (!Double.isNaN(f0Array[j])) { // a valid value
+					if (iLastValid == j - 1) {
+						// no need to interpolate
+						f0AndInterpol[j] = f0Array[j];
+					} else {
+						// need to interpolate
+						double prevF0;
+						if (iLastValid < 0) { // we don't have a previous value -- use current one
+							prevF0 = f0Array[j];
+						} else {
+							prevF0 = f0Array[iLastValid];
+						}
+						double delta = (f0Array[j] - prevF0) / (j - iLastValid);
+						double f0 = prevF0;
+						for (int k = iLastValid + 1; k < j; k++) {
+							f0 += delta;
+							interpol[k] = f0;
+							f0AndInterpol[k] = f0;
+						}
+					}
+					iLastValid = j;
+				}
+			}
+			f0Graph.addDataSeries(interpol, Color.GREEN, FunctionGraph.DRAW_DOTS, FunctionGraph.DOT_EMPTYCIRCLE);
+			jf.repaint();
+
+			double[] f0AndInterpolate = combineF0andInterpolate(f0Array, interpol);
+
+			// double[] coeffs = Polynomial.fitPolynomial(sylF0, polynomOrder);
+			int polynomialOrder = (new Integer(getProp(POLYORDER))).intValue();
+			double[] coeffs = Polynomial.fitPolynomial(f0AndInterpolate, polynomialOrder);
+
+			if (coeffs != null) {
+				double[] sylPred = Polynomial.generatePolynomialValues(coeffs, interpol.length, 0, 1);
+				f0Graph.addDataSeries(sylPred, Color.RED, FunctionGraph.DRAW_LINE, -1);
+				double eqDistance = DistanceComputer.getEuclideanDistance(sylPred, f0AndInterpolate);
+				System.out.println(baseName + " - EqDist: " + eqDistance / (double) sylPred.length);
+				costMeasure += (eqDistance / (double) sylPred.length);
+				featurePW.print(baseName + " ");
+				for (double c : coeffs) {
+					featurePW.print(c + " ");
+				}
+				featurePW.println();
+
+			}
+
+			// double[] pred = Polynomial.generatePolynomialValues(coeffs, iUnitDurationInArray, 0, 1);
+			// f0Graph.addDataSeries(unitF0, Color.BLACK, FunctionGraph.DRAW_LINE, -1);
+
+			try {
+				ap.join();
+				Thread.sleep(10);
+			} catch (InterruptedException ie) {
+			}
+			// System.out.println();
+
+		}
+
+	}
+
+	private double[] cutStartEndUnvoicedSegments(double[] array) {
+
+		if (array == null)
+			return null;
+
+		int startIndex = 0;
+		int endIndex = array.length;
+
+		// find start index
+		for (int i = 0; i < array.length; i++) {
+			if (array[i] != 0) {
+				startIndex = i;
+				break;
+			}
+		}
+
+		// find end index
+		for (int i = (array.length - 1); i > 0; i--) {
+			if (array[i] != 0) {
+				endIndex = i;
+				break;
+			}
+		}
+
+		int newArraySize = endIndex - startIndex;
+
+		double[] newArray = new double[newArraySize];
+		System.arraycopy(array, startIndex, newArray, 0, newArraySize);
+
+		for (int i = 0; i < newArray.length; i++) {
+			System.out.println(newArray[i]);
+		}
+		System.out.println("Resized from " + array.length + " to " + newArraySize);
+
+		return newArray;
+	}
+
+	private String getCharacterName(String baseName) {
+		Iterator<String> it = characters.iterator();
+		while (it.hasNext()) {
+			String character = it.next().trim();
+			if (baseName.startsWith(character)) {
+				return character;
+			}
+		}
+		return null;
+	}
+
+	private double[] combineF0andInterpolate(double[] f0Array, double[] interpol) {
+
+		double[] f0AndInterpolate = new double[f0Array.length];
+		Arrays.fill(f0AndInterpolate, Double.NaN);
+		for (int i = 0; i < f0Array.length; i++) {
+			if (!Double.isNaN(f0Array[i])) {
+				f0AndInterpolate[i] = f0Array[i];
+			} else if (!Double.isNaN(interpol[i])) {
+				f0AndInterpolate[i] = interpol[i];
+			}
+			// System.out.println(f0Array[i]+" "+interpol[i]+" "+f0AndInterpolate[i]);
+		}
+
+		return f0AndInterpolate;
+	}
+
+	/**
+	 * Provide the progress of computation, in percent, or -1 if that feature is not implemented.
+	 * 
+	 * @return -1 if not implemented, or an integer between 0 and 100.
+	 */
+	public int getProgress() {
+		return percent;
+	}
+
+	/**
+	 * @param args
+	 */
+	public static void main(String[] args) throws Exception {
+		VocalizationF0PolynomialInspector acfeatsWriter = new VocalizationF0PolynomialInspector();
+		DatabaseLayout db = new DatabaseLayout(acfeatsWriter);
+		acfeatsWriter.compute();
+	}
 
 }
-

marytts-builder/src/main/java/marytts/tools/voiceimport/vocalizations/VocalizationFeatureFileWriter.java

@@ -48,242 +48,231 @@ import marytts.vocalizations.VocalizationUnitFileReader;
 
 /**
  * vocalization feature file writer
+ * 
  * @author sathish
  *
  */
-public class VocalizationFeatureFileWriter extends VoiceImportComponent
-{
-    
-    protected File outFeatureFile;
-    protected FeatureDefinition featureDefinition;
-    protected VocalizationUnitFileReader listenerUnits;
-    
-    protected DatabaseLayout db = null;
-    protected int percent = 0;
-    private ArrayList<String> featureCategories; // feature categories  
-    private Map<String,Map<String, String>> annotationData; // basename --> (feature category, feature value)
-    
-    protected String vocalizationsDir;
-    protected BasenameList bnlVocalizations;
-    
-    private final String name = "VocalizationFeatureFileWriter";
-    public final String UNITFILE = name + ".unitFile";
-    public final String FEATUREFILE = name + ".featureFile";
-    public final String MANUALFEATURES = name + ".annotationFeatureFile";  
-    public final String FEATDEF = name + ".featureDefinition";
-    
-    public final String POLYNOMORDER = name + ".polynomOrder";
-    public final String SHOWGRAPH = name + ".showGraph";
-    public final String INTERPOLATE = name + ".interpolate";
-    public final String MINPITCH = name + ".minPitch";
-    public final String MAXPITCH = name + ".maxPitch";
-    
-    //public String BASELIST =  name + ".backchannelBNL";
- 
-    public String getName(){
-        return name;
-    }
-   
-    
-   public SortedMap<String, String> getDefaultProps(DatabaseLayout db){
-       this.db = db;
-       if (props == null){
-           props = new TreeMap<String, String>();
-           String fileDir = db.getProp(db.VOCALIZATIONSDIR)+File.separator+"files"+File.separator;
-           String maryExt = db.getProp(db.MARYEXT);
-           props.put(UNITFILE,fileDir+"vocalization_units"+maryExt);
-           props.put(FEATUREFILE,fileDir+"vocalization_features"+maryExt);
-           props.put(MANUALFEATURES,db.getProp(db.VOCALIZATIONSDIR)+File.separator+"features"
-                   +File.separator+"annotation_vocalizations_features.txt");
-           props.put(FEATDEF,db.getProp(db.VOCALIZATIONSDIR)+File.separator+"features"
-                   +File.separator+"vocalization_feature_definition.txt");
-       }
-       return props;
-   }
-    
-   protected void setupHelp()
-   {
-       if (props2Help ==null) {
-           props2Help = new TreeMap<String, String>();
-           props2Help.put(UNITFILE,"file containing all halfphone units");
-           props2Help.put(FEATUREFILE,"file containing all halfphone units and their target cost features");
-       }
-   }
-   
-   @Override
-   protected void initialiseComp()
-   {
-       
-       featureCategories = new ArrayList<String>(); // feature categories  
-       annotationData = new HashMap<String,Map<String, String>>(); // basename --> (feature category, feature value)
+public class VocalizationFeatureFileWriter extends VoiceImportComponent {
 
-       try {
-           String basenameFile = db.getProp(db.VOCALIZATIONSDIR)+File.separator+"basenames.lst";
-           if ( (new File(basenameFile)).exists() ) {
-               System.out.println("Loading basenames of vocalizations from '"+basenameFile+"' list...");
-               bnlVocalizations = new BasenameList(basenameFile);
-               System.out.println("Found "+bnlVocalizations.getLength()+ " vocalizations in basename list");
-           }
-           else {
-               String vocalWavDir = db.getProp(db.VOCALIZATIONSDIR)+File.separator+"wav";
-               System.out.println("Loading basenames of vocalizations from '"+vocalWavDir+"' directory...");
-               bnlVocalizations = new BasenameList(vocalWavDir, ".wav");
-               System.out.println("Found "+bnlVocalizations.getLength()+ " vocalizations in "+ vocalWavDir + " directory");
-           }
-       } catch (IOException e) {
-           e.printStackTrace();
-       }
-   }
-  
-    @Override
-    public boolean compute() throws IOException, MaryConfigurationException
-    {
-        // read feature definition
-        BufferedReader fDBfr = new BufferedReader(new FileReader(new File(getProp(FEATDEF))));
-        //featureDefinition = new FeatureDefinition(fDBfr, true);
-        featureDefinition = new FeatureDefinition(fDBfr, true);
-        listenerUnits = new VocalizationUnitFileReader(getProp(UNITFILE));
-        
-        // load annotation
-        VocalizationAnnotationReader annotationReader = new VocalizationAnnotationReader(getProp(MANUALFEATURES), bnlVocalizations);
-        annotationData = annotationReader.getVocalizationsAnnotation();
-        featureCategories = annotationReader.getFeatureList();
-        
-        // write features into timeline file
-        DataOutputStream out = new DataOutputStream(new BufferedOutputStream(new FileOutputStream(new File(getProp(FEATUREFILE)))));
-        writeHeaderTo(out);
-        writeUnitFeaturesTo(out);
-        out.close();
-        logger.debug("Number of processed units: " + listenerUnits.getNumberOfUnits() );
+	protected File outFeatureFile;
+	protected FeatureDefinition featureDefinition;
+	protected VocalizationUnitFileReader listenerUnits;
 
-        VocalizationFeatureFileReader tester = new VocalizationFeatureFileReader(getProp(FEATUREFILE));
-        int unitsOnDisk = tester.getNumberOfUnits();
-        if (unitsOnDisk == listenerUnits.getNumberOfUnits()) {
-            System.out.println("Can read right number of units");
-            return true;
-        } else {
-            System.out.println("Read wrong number of units: "+unitsOnDisk);
-            return false;
-        }
-    }
+	protected DatabaseLayout db = null;
+	protected int percent = 0;
+	private ArrayList<String> featureCategories; // feature categories
+	private Map<String, Map<String, String>> annotationData; // basename --> (feature category, feature value)
 
-    protected void writeUnitFeaturesTo(DataOutput out) throws IOException, UnsupportedEncodingException, FileNotFoundException {
-        
-        int numUnits = listenerUnits.getNumberOfUnits();
-        
-        out.writeInt( numUnits );
-        //logger.debug("Number of vocalization units : "+numUnits);
-        System.out.println("Number of vocalization units : "+numUnits);
-        System.out.println("Annotation number of units: "+annotationData.size());
-        if ( annotationData.size() != listenerUnits.getNumberOfUnits() ) {
-            throw new RuntimeException("Number of units in vocalizations unit-file is not equal to number of basenames. ");
-        }
-        
-        if ( featureCategories.size() != featureDefinition.getNumberOfFeatures() ) {
-            throw new RuntimeException("Number of categories in feature_definition is not equal to features given in annotation file ");
-        }
-        
-        /**
-         * TODO sanity check for each basename
-         */
-        
-        int noOfFeatures = featureDefinition.getNumberOfFeatures();
-        String[] featureNames = new String[noOfFeatures];
-        for( int i=0; i < featureNames.length; i++ ) {
-            featureNames[i] = featureDefinition.getFeatureName(i);
-        }
-        
-        for( int i=0; i < bnlVocalizations.getLength(); i++ ) {
-            
-            byte[]  byteFeatures;
-            short[] shortFeatures;
-            float[] continiousFeatures;
-            
-            String baseName = bnlVocalizations.getName(i);
-            Map<String, String> singleAnnotation = annotationData.get(baseName);
-            
-            int noByteFeatures = featureDefinition.getNumberOfByteFeatures();
-            int noShortFeatures = featureDefinition.getNumberOfShortFeatures();
-            int noContiniousFeatures = featureDefinition.getNumberOfContinuousFeatures();
-            
-            // create features array
-            byteFeatures  = new byte[noByteFeatures];
-            shortFeatures = new short[noShortFeatures];
-            continiousFeatures = new float[noContiniousFeatures];
-            
-            int countByteFeatures = 0;
-            int countShortFeatures = 0;
-            int countFloatFeatures = 0;
-            
-            for ( int j=0; j < featureNames.length; j++ ) {
-                String fName = featureNames[j];
-                if ( featureDefinition.isByteFeature(j) ) { // Byte feature
-                    if ( singleAnnotation.containsKey(fName) ) {
-                        byteFeatures[countByteFeatures++] = featureDefinition.getFeatureValueAsByte(j, singleAnnotation.get(fName));
-                    }
-                    else {
-                        byteFeatures[countByteFeatures++] = (byte) 0;
-                    }
-                }
-                else if ( featureDefinition.isShortFeature(j) ) { // Short feature
-                    if ( singleAnnotation.containsKey(fName) ) {
-                        shortFeatures[countShortFeatures++] = featureDefinition.getFeatureValueAsShort(j, singleAnnotation.get(fName));
-                    }
-                    else {
-                        shortFeatures[countShortFeatures++] = (short) 0;
-                    }
-                }
-                else if ( featureDefinition.isContinuousFeature(j) ) { // Continuous feature
-                    if ( !singleAnnotation.containsKey(fName) ) {
-                        continiousFeatures[countFloatFeatures++] = Float.NaN;
-                    }
-                    else if ( "NRI".equals(singleAnnotation.get(fName)) ) {
-                        continiousFeatures[countFloatFeatures++] = Float.NaN;
-                    }
-                    else {
-                        continiousFeatures[countFloatFeatures++] = (new Float(singleAnnotation.get(fName))).floatValue();
-                    }
-                }
-            }
-            
-            
-            FeatureVector outFV = featureDefinition.toFeatureVector(i, byteFeatures, shortFeatures, continiousFeatures);
-            outFV.writeTo(out);
-        }
-        
-    }
-    
-    /**
-     * Write the header of this feature file to the given DataOutput
-     * @param out
-     * @throws IOException
-     */
-    protected void writeHeaderTo(DataOutput out) throws IOException {
-        new MaryHeader(MaryHeader.LISTENERFEATS).writeTo(out);
-        featureDefinition.writeBinaryTo(out);
-    }
+	protected String vocalizationsDir;
+	protected BasenameList bnlVocalizations;
 
-    /**
-     * Provide the progress of computation, in percent, or -1 if
-     * that feature is not implemented.
-     * @return -1 if not implemented, or an integer between 0 and 100.
-     */
-    public int getProgress()
-    {
-        return percent;
-    }
+	private final String name = "VocalizationFeatureFileWriter";
+	public final String UNITFILE = name + ".unitFile";
+	public final String FEATUREFILE = name + ".featureFile";
+	public final String MANUALFEATURES = name + ".annotationFeatureFile";
+	public final String FEATDEF = name + ".featureDefinition";
 
-    /**
-     * @param args
-     */
-    public static void main(String[] args) throws Exception
-    {
-        VocalizationFeatureFileWriter acfeatsWriter = 
-            new VocalizationFeatureFileWriter();
-        DatabaseLayout db = new DatabaseLayout(acfeatsWriter);
-        acfeatsWriter.compute();
-    }
+	public final String POLYNOMORDER = name + ".polynomOrder";
+	public final String SHOWGRAPH = name + ".showGraph";
+	public final String INTERPOLATE = name + ".interpolate";
+	public final String MINPITCH = name + ".minPitch";
+	public final String MAXPITCH = name + ".maxPitch";
 
+	// public String BASELIST = name + ".backchannelBNL";
+
+	public String getName() {
+		return name;
+	}
+
+	public SortedMap<String, String> getDefaultProps(DatabaseLayout db) {
+		this.db = db;
+		if (props == null) {
+			props = new TreeMap<String, String>();
+			String fileDir = db.getProp(db.VOCALIZATIONSDIR) + File.separator + "files" + File.separator;
+			String maryExt = db.getProp(db.MARYEXT);
+			props.put(UNITFILE, fileDir + "vocalization_units" + maryExt);
+			props.put(FEATUREFILE, fileDir + "vocalization_features" + maryExt);
+			props.put(MANUALFEATURES, db.getProp(db.VOCALIZATIONSDIR) + File.separator + "features" + File.separator
+					+ "annotation_vocalizations_features.txt");
+			props.put(FEATDEF, db.getProp(db.VOCALIZATIONSDIR) + File.separator + "features" + File.separator
+					+ "vocalization_feature_definition.txt");
+		}
+		return props;
+	}
+
+	protected void setupHelp() {
+		if (props2Help == null) {
+			props2Help = new TreeMap<String, String>();
+			props2Help.put(UNITFILE, "file containing all halfphone units");
+			props2Help.put(FEATUREFILE, "file containing all halfphone units and their target cost features");
+		}
+	}
+
+	@Override
+	protected void initialiseComp() {
+
+		featureCategories = new ArrayList<String>(); // feature categories
+		annotationData = new HashMap<String, Map<String, String>>(); // basename --> (feature category, feature value)
+
+		try {
+			String basenameFile = db.getProp(db.VOCALIZATIONSDIR) + File.separator + "basenames.lst";
+			if ((new File(basenameFile)).exists()) {
+				System.out.println("Loading basenames of vocalizations from '" + basenameFile + "' list...");
+				bnlVocalizations = new BasenameList(basenameFile);
+				System.out.println("Found " + bnlVocalizations.getLength() + " vocalizations in basename list");
+			} else {
+				String vocalWavDir = db.getProp(db.VOCALIZATIONSDIR) + File.separator + "wav";
+				System.out.println("Loading basenames of vocalizations from '" + vocalWavDir + "' directory...");
+				bnlVocalizations = new BasenameList(vocalWavDir, ".wav");
+				System.out.println("Found " + bnlVocalizations.getLength() + " vocalizations in " + vocalWavDir + " directory");
+			}
+		} catch (IOException e) {
+			e.printStackTrace();
+		}
+	}
+
+	@Override
+	public boolean compute() throws IOException, MaryConfigurationException {
+		// read feature definition
+		BufferedReader fDBfr = new BufferedReader(new FileReader(new File(getProp(FEATDEF))));
+		// featureDefinition = new FeatureDefinition(fDBfr, true);
+		featureDefinition = new FeatureDefinition(fDBfr, true);
+		listenerUnits = new VocalizationUnitFileReader(getProp(UNITFILE));
+
+		// load annotation
+		VocalizationAnnotationReader annotationReader = new VocalizationAnnotationReader(getProp(MANUALFEATURES),
+				bnlVocalizations);
+		annotationData = annotationReader.getVocalizationsAnnotation();
+		featureCategories = annotationReader.getFeatureList();
+
+		// write features into timeline file
+		DataOutputStream out = new DataOutputStream(
+				new BufferedOutputStream(new FileOutputStream(new File(getProp(FEATUREFILE)))));
+		writeHeaderTo(out);
+		writeUnitFeaturesTo(out);
+		out.close();
+		logger.debug("Number of processed units: " + listenerUnits.getNumberOfUnits());
+
+		VocalizationFeatureFileReader tester = new VocalizationFeatureFileReader(getProp(FEATUREFILE));
+		int unitsOnDisk = tester.getNumberOfUnits();
+		if (unitsOnDisk == listenerUnits.getNumberOfUnits()) {
+			System.out.println("Can read right number of units");
+			return true;
+		} else {
+			System.out.println("Read wrong number of units: " + unitsOnDisk);
+			return false;
+		}
+	}
+
+	protected void writeUnitFeaturesTo(DataOutput out) throws IOException, UnsupportedEncodingException, FileNotFoundException {
+
+		int numUnits = listenerUnits.getNumberOfUnits();
+
+		out.writeInt(numUnits);
+		// logger.debug("Number of vocalization units : "+numUnits);
+		System.out.println("Number of vocalization units : " + numUnits);
+		System.out.println("Annotation number of units: " + annotationData.size());
+		if (annotationData.size() != listenerUnits.getNumberOfUnits()) {
+			throw new RuntimeException("Number of units in vocalizations unit-file is not equal to number of basenames. ");
+		}
+
+		if (featureCategories.size() != featureDefinition.getNumberOfFeatures()) {
+			throw new RuntimeException(
+					"Number of categories in feature_definition is not equal to features given in annotation file ");
+		}
+
+		/**
+		 * TODO sanity check for each basename
+		 */
+
+		int noOfFeatures = featureDefinition.getNumberOfFeatures();
+		String[] featureNames = new String[noOfFeatures];
+		for (int i = 0; i < featureNames.length; i++) {
+			featureNames[i] = featureDefinition.getFeatureName(i);
+		}
+
+		for (int i = 0; i < bnlVocalizations.getLength(); i++) {
+
+			byte[] byteFeatures;
+			short[] shortFeatures;
+			float[] continiousFeatures;
+
+			String baseName = bnlVocalizations.getName(i);
+			Map<String, String> singleAnnotation = annotationData.get(baseName);
+
+			int noByteFeatures = featureDefinition.getNumberOfByteFeatures();
+			int noShortFeatures = featureDefinition.getNumberOfShortFeatures();
+			int noContiniousFeatures = featureDefinition.getNumberOfContinuousFeatures();
+
+			// create features array
+			byteFeatures = new byte[noByteFeatures];
+			shortFeatures = new short[noShortFeatures];
+			continiousFeatures = new float[noContiniousFeatures];
+
+			int countByteFeatures = 0;
+			int countShortFeatures = 0;
+			int countFloatFeatures = 0;
+
+			for (int j = 0; j < featureNames.length; j++) {
+				String fName = featureNames[j];
+				if (featureDefinition.isByteFeature(j)) { // Byte feature
+					if (singleAnnotation.containsKey(fName)) {
+						byteFeatures[countByteFeatures++] = featureDefinition.getFeatureValueAsByte(j,
+								singleAnnotation.get(fName));
+					} else {
+						byteFeatures[countByteFeatures++] = (byte) 0;
+					}
+				} else if (featureDefinition.isShortFeature(j)) { // Short feature
+					if (singleAnnotation.containsKey(fName)) {
+						shortFeatures[countShortFeatures++] = featureDefinition.getFeatureValueAsShort(j,
+								singleAnnotation.get(fName));
+					} else {
+						shortFeatures[countShortFeatures++] = (short) 0;
+					}
+				} else if (featureDefinition.isContinuousFeature(j)) { // Continuous feature
+					if (!singleAnnotation.containsKey(fName)) {
+						continiousFeatures[countFloatFeatures++] = Float.NaN;
+					} else if ("NRI".equals(singleAnnotation.get(fName))) {
+						continiousFeatures[countFloatFeatures++] = Float.NaN;
+					} else {
+						continiousFeatures[countFloatFeatures++] = (new Float(singleAnnotation.get(fName))).floatValue();
+					}
+				}
+			}
+
+			FeatureVector outFV = featureDefinition.toFeatureVector(i, byteFeatures, shortFeatures, continiousFeatures);
+			outFV.writeTo(out);
+		}
+
+	}
+
+	/**
+	 * Write the header of this feature file to the given DataOutput
+	 * 
+	 * @param out
+	 * @throws IOException
+	 */
+	protected void writeHeaderTo(DataOutput out) throws IOException {
+		new MaryHeader(MaryHeader.LISTENERFEATS).writeTo(out);
+		featureDefinition.writeBinaryTo(out);
+	}
+
+	/**
+	 * Provide the progress of computation, in percent, or -1 if that feature is not implemented.
+	 * 
+	 * @return -1 if not implemented, or an integer between 0 and 100.
+	 */
+	public int getProgress() {
+		return percent;
+	}
+
+	/**
+	 * @param args
+	 */
+	public static void main(String[] args) throws Exception {
+		VocalizationFeatureFileWriter acfeatsWriter = new VocalizationFeatureFileWriter();
+		DatabaseLayout db = new DatabaseLayout(acfeatsWriter);
+		acfeatsWriter.compute();
+	}
 
 }
-

marytts-builder/src/main/java/marytts/tools/voiceimport/vocalizations/VocalizationIntonationWriter.java

@@ -51,388 +51,385 @@ import marytts.util.signal.SignalProcUtils;
 import marytts.vocalizations.VocalizationIntonationReader;
 import marytts.vocalizations.VocalizationUnitFileReader;
 
-
 /**
- * Vocalization intonation writer into a time-line file
- * This class can create a timeline file with intonation contours and thier polynomial coeffs 
+ * Vocalization intonation writer into a time-line file This class can create a timeline file with intonation contours and thier
+ * polynomial coeffs
  * 
  * @author sathish pammi
  *
  */
 public class VocalizationIntonationWriter extends VoiceImportComponent {
 
-    protected String vocalizationsDir;
-    protected BasenameList bnlVocalizations;
-    protected VocalizationUnitFileReader listenerUnits;
-    
-    protected DatabaseLayout db = null;
-    protected int percent = 0;
-    
-    public final String PITCHDIR = getName()+".pitchDir";
-    public final String WAVEDIR = getName()+".inputWaveDir";
-    public final String POLYORDER = getName()+".polynomialOrder";
-    public final String ISEXTERNALF0 = getName()+".isExternalF0Usage";
-    public final String EXTERNALF0FORMAT = getName()+".externalF0Format";
-    public final String EXTERNALEXT = getName()+".externalF0Extention";
-    public final String UNITFILE = getName()+".unitFile";
-    public final String SKIPSIZE = getName()+".skipSize";
-    public final String WINDOWSIZE = getName()+".windowSize";
-    public final String F0TIMELINE = getName()+".intonationTimeLineFile";
-    public final String F0FEATDEF = getName()+".intonationFeatureDefinition";
-    
-    public String getName() {
-        return "VocalizationIntonationWriter";
-    }
-    
-    @Override
-    protected void initialiseComp() {
-        
-        String timelineDir = db.getProp(db.VOCALIZATIONSDIR) + File.separator + "files";
-        if (!(new File(timelineDir)).exists()) {
-            System.out.println("vocalizations/files directory does not exist; ");
-            if (!(new File(timelineDir)).mkdirs()) {
-                throw new Error("Could not create vocalizations/files");
-            }
-            System.out.println("Created successfully.\n");
-        }
-        
-        try {
-            String basenameFile = db.getProp(db.VOCALIZATIONSDIR)+File.separator+"basenames.lst";
-            if ( (new File(basenameFile)).exists() ) {
-                System.out.println("Loading basenames of vocalizations from '"+basenameFile+"' list...");
-                bnlVocalizations = new BasenameList(basenameFile);
-                System.out.println("Found "+bnlVocalizations.getLength()+ " vocalizations in basename list");
-            }
-            else {
-                String vocalWavDir = db.getProp(db.VOCALIZATIONSDIR)+File.separator+"wav";
-                System.out.println("Loading basenames of vocalizations from '"+vocalWavDir+"' directory...");
-                bnlVocalizations = new BasenameList(vocalWavDir, ".wav");
-                System.out.println("Found "+bnlVocalizations.getLength()+ " vocalizations in "+ vocalWavDir + " directory");
-            }
-        } catch (IOException e) {
-            e.printStackTrace();
-        }
-    }
-    
-    public SortedMap getDefaultProps(DatabaseLayout db) {
-       this.db = db;
-       if (props == null){
-           props = new TreeMap<String, String>();
-           props.put(WAVEDIR,db.getProp(db.VOCALIZATIONSDIR)+File.separator+"wav");
-           props.put(UNITFILE,db.getProp(db.VOCALIZATIONSDIR)+File.separator+"files"+File.separator+"vocalization_units"+db.getProp(db.MARYEXT));
-           props.put(POLYORDER,"3");
-           props.put(ISEXTERNALF0,"true");
-           props.put(EXTERNALF0FORMAT,"sptk");
-           props.put(EXTERNALEXT, ".lf0");
-           props.put(PITCHDIR,db.getProp(db.VOCALIZATIONSDIR)+File.separator+"lf0");
-           props.put(SKIPSIZE, "0.005");
-           props.put(WINDOWSIZE, "0.005");
-           props.put(F0TIMELINE, db.getProp(db.VOCALIZATIONSDIR)+File.separator+"files"+File.separator+"vocalization_intonation"+db.getProp(db.MARYEXT));
-           props.put(F0FEATDEF, db.getProp(db.VOCALIZATIONSDIR)+File.separator+"features"+File.separator+"vocalization_f0_feature_definition.txt");
-       }
-       return props;
-    }
-    
-    protected void setupHelp() {
-        props2Help = new TreeMap();
-        
-    }
-    
-    /**
-     *  Reads and concatenates a list of waveforms into one single timeline file.
-     * @throws IOException 
-     */
-    @Override
-    public boolean compute() throws IOException, MaryConfigurationException {
-        
-        listenerUnits = new VocalizationUnitFileReader(getProp(UNITFILE));
-        
-        // write features into timeline file
-        DataOutputStream out = new DataOutputStream(new BufferedOutputStream(new FileOutputStream(new File(getProp(F0TIMELINE)))));
-        writeHeaderTo(out);
-        writeUnitFeaturesTo(out);
-        out.close();
-    
-        VocalizationIntonationReader tester = new VocalizationIntonationReader(getProp(F0TIMELINE));
-        int unitsOnDisk = tester.getNumberOfUnits();
-        if (unitsOnDisk == listenerUnits.getNumberOfUnits()) {
-            System.out.println("Can read right number of units");
-            return true;
-        } else {
-            System.out.println("Read wrong number of units: "+unitsOnDisk);
-            return false;
-        }
-    }    
-    
-    /**
-     * 
-     * @param out
-     * @throws IOException
-     */
-    protected void writeUnitFeaturesTo(DataOutput out) throws IOException {
-        
-        int numUnits = listenerUnits.getNumberOfUnits();
-        float windowSize = new Float(getProp(WINDOWSIZE)).floatValue();
-        float skipSize = new Float(getProp(SKIPSIZE)).floatValue();
-        
-        out.writeFloat(windowSize);
-        out.writeFloat(skipSize);
-        out.writeInt( numUnits );
-        
-        
-        for( int i=0; i < bnlVocalizations.getLength(); i++ ) {
-            
-            double[] f0Array = null;
-            
-            try {
-                f0Array = getVocalizationF0(bnlVocalizations.getName(i), false);
-            } catch (UnsupportedAudioFileException e) {
-                e.printStackTrace();
-            }
-            
-            // write coeffs followed by its order
-            double[] coeffs = getPolynomialCoeffs(f0Array);
-            if ( coeffs == null ) {
-                out.writeInt(0);
-            }
-            else {
-                out.writeInt(coeffs.length);
-                for ( int j=0; j < coeffs.length; j++ ) {
-                    out.writeFloat((float)coeffs[j]);
-                }
-            } 
-            
-            // write f0 Array followed by f0 contour array size
-            if ( f0Array == null ) {
-                out.writeInt(0);
-            }
-            else {
-                out.writeInt(f0Array.length);
-                for ( int j=0; j < f0Array.length; j++ ) {
-                    out.writeFloat((float)f0Array[j]);
-                }
-            }  
-        }
-    }
- 
-    /**
-     * get f0 contour of vocalization f0
-     * @param baseName
-     * @return
-     * @throws UnsupportedAudioFileException
-     * @throws IOException
-     */
-     private double[] getVocalizationF0(String baseName, boolean doInterpolate) throws UnsupportedAudioFileException, IOException {
-     
-         double[] f0Array = null;
-         
-         if ( "true".equals(getProp(ISEXTERNALF0)) ) {
-             
-             String externalFormat = getProp(EXTERNALF0FORMAT);
-             String externalExt    = getProp(EXTERNALEXT);
-             System.out.println("Loading f0 contour from file : "+ getProp(PITCHDIR) + File.separator + baseName + externalExt);
-             if ( "sptk".equals(externalFormat) ) {
-                 String fileName = getProp(PITCHDIR) + File.separator + baseName + externalExt;
-                 SPTKPitchReaderWriter sprw = new SPTKPitchReaderWriter(fileName);
-                 f0Array = sprw.getF0Contour();
-             }
-             else if ( "ptc".equals(externalFormat) ) {
-                 String fileName = getProp(PITCHDIR) + File.separator + baseName + externalExt;
-                 PitchReaderWriter sprw = new PitchReaderWriter(fileName);
-                 f0Array = sprw.contour;
-             }
-         }
-         else {
-             PitchFileHeader params = new PitchFileHeader();
-             F0TrackerAutocorrelationHeuristic tracker = new F0TrackerAutocorrelationHeuristic(params);
-             String waveFile = db.getProp(db.VOCALIZATIONSDIR)+File.separator+"wav"+baseName+db.getProp(db.WAVEXT);
-             System.out.println("Computing f0 contour from wave file: "+ waveFile);
-             AudioInputStream inputAudio = AudioSystem.getAudioInputStream(new File(waveFile));
-             
-             // Enforce PCM_SIGNED encoding
-             if (!inputAudio.getFormat().getEncoding().equals(AudioFormat.Encoding.PCM_SIGNED)) {
-                 inputAudio = AudioSystem.getAudioInputStream(AudioFormat.Encoding.PCM_SIGNED, inputAudio);
-             }
-             
-             int audioSampleRate = (int)inputAudio.getFormat().getSampleRate();
-             AudioDoubleDataSource signal = new AudioDoubleDataSource(inputAudio);
-             double [] sentenceAudio = signal.getAllData();
-             tracker.pitchAnalyze(new BufferedDoubleDataSource(sentenceAudio));
-             //double frameShiftTime = tracker.getSkipSizeInSeconds();
-             f0Array = tracker.getF0Contour();
-         }
-         
-         if ( doInterpolate ) {
-             return interpolateF0Array(f0Array); 
-         }
-         
-         return f0Array;
-     }
-     
-     
-   /**
-    * to get polynomial coeffs of f0 contour
-    * @param f0Array
-    * @return
-    */
-   private double[] getPolynomialCoeffs(double[] f0Array) {
-       
-       if ( f0Array == null ) {
-           return null;
-       }
-       
-       f0Array = cutStartEndUnvoicedSegments(f0Array);
-       double[] f0AndInterpolate = interpolateF0Array(f0Array);
-       int polynomialOrder = (new Integer(getProp(POLYORDER))).intValue();
-       double[] coeffs = Polynomial.fitPolynomial(f0AndInterpolate, polynomialOrder);
-       return coeffs;
-   }
+	protected String vocalizationsDir;
+	protected BasenameList bnlVocalizations;
+	protected VocalizationUnitFileReader listenerUnits;
 
-   /**
-    * to interpolate F0 contour values
-    * @param f0Array
-    * @return
-    */
-   private double[] interpolateF0Array(double[] f0Array) {
-       
-       if (f0Array == null) {
-           return null;
-       }
-       
-       for (int j=0; j<f0Array.length; j++) {
-           if (f0Array[j] == 0) {
-               f0Array[j] = Double.NaN;
-           }
-       }
-       if (f0Array.length >= 3) {
-           f0Array = SignalProcUtils.medianFilter(f0Array, 5);
-       }
-       
-       double[] f0AndInterpol;
-       double[] interpol = new double[f0Array.length];
-       Arrays.fill(interpol, Double.NaN);
-       f0AndInterpol = new double[f0Array.length];
-       int iLastValid = -1;
-       for (int j=0; j<f0Array.length; j++) {
-           if (!Double.isNaN(f0Array[j])) { // a valid value
-               if (iLastValid == j-1) {
-                   // no need to interpolate
-                   f0AndInterpol[j] = f0Array[j];
-               } else {
-                   // need to interpolate
-                   double prevF0;
-                   if (iLastValid < 0) { // we don't have a previous value -- use current one
-                       prevF0 = f0Array[j];
-                   } else {
-                       prevF0 = f0Array[iLastValid];
-                   }
-                   double delta = (f0Array[j]-prevF0)/(j-iLastValid);
-                   double f0 = prevF0;
-                   for (int k = iLastValid+1; k<j; k++) {
-                       f0 += delta;
-                       interpol[k] = f0;
-                       f0AndInterpol[k] = f0;
-                   }
-               }
-               iLastValid = j;
-           }
-       }
-       
-       double[] f0AndInterpolate = combineF0andInterpolate(f0Array, interpol);
-       return f0AndInterpolate;
-   }
+	protected DatabaseLayout db = null;
+	protected int percent = 0;
 
-    
-    /**
-     * cut  begin-end unvoiced segments 
-     * @param array
-     * @return
-     */
-    private double[] cutStartEndUnvoicedSegments(double[] array) {
-        
-        if(array == null) return null;
-            
-        int startIndex = 0;
-        int endIndex   = array.length;
-        
-        // find start index
-        for ( int i=0; i<array.length; i++ ) {
-            if (array[i] != 0){
-                startIndex = i;
-                break;
-            }
-        }
-        
-        // find end index
-        for ( int i=(array.length - 1); i>0 ; i-- ) {
-            if (array[i] != 0){
-                endIndex = i;
-                break;
-            }
-        }
-        
-        int newArraySize = endIndex - startIndex;
-        
-        
-        double[] newArray = new double[newArraySize];
-        System.arraycopy(array, startIndex, newArray, 0, newArraySize);
-        
-        /*for ( int i=0; i<newArray.length; i++ ) {
-            System.out.println(newArray[i]);
-        }
-        System.out.println("Resized from "+array.length+" to "+newArraySize);
-        */
-        return newArray;
-    }
-    
-    /**
-     * interpolate f0
-     * @param f0Array
-     * @param interpol
-     * @return
-     */
-    private double[] combineF0andInterpolate(double[] f0Array, double[] interpol) {
-        
-        double[] f0AndInterpolate = new double[f0Array.length];
-        Arrays.fill(f0AndInterpolate, Double.NaN);
-        for(int i=0; i<f0Array.length; i++) {
-            if( !Double.isNaN(f0Array[i]) ) {
-                f0AndInterpolate[i] = f0Array[i];
-            }
-            else if ( !Double.isNaN(interpol[i]) ) {
-                f0AndInterpolate[i] = interpol[i];
-            }
-            //System.out.println(f0Array[i]+" "+interpol[i]+" "+f0AndInterpolate[i]);
-        }
-        return f0AndInterpolate;
-    }
+	public final String PITCHDIR = getName() + ".pitchDir";
+	public final String WAVEDIR = getName() + ".inputWaveDir";
+	public final String POLYORDER = getName() + ".polynomialOrder";
+	public final String ISEXTERNALF0 = getName() + ".isExternalF0Usage";
+	public final String EXTERNALF0FORMAT = getName() + ".externalF0Format";
+	public final String EXTERNALEXT = getName() + ".externalF0Extention";
+	public final String UNITFILE = getName() + ".unitFile";
+	public final String SKIPSIZE = getName() + ".skipSize";
+	public final String WINDOWSIZE = getName() + ".windowSize";
+	public final String F0TIMELINE = getName() + ".intonationTimeLineFile";
+	public final String F0FEATDEF = getName() + ".intonationFeatureDefinition";
 
-    
-    /**
-     * Write the header of this feature file to the given DataOutput
-     * @param out
-     * @throws IOException
-     */
-    protected void writeHeaderTo(DataOutput out) throws IOException {
-        new MaryHeader(MaryHeader.LISTENERFEATS).writeTo(out);
-    }
+	public String getName() {
+		return "VocalizationIntonationWriter";
+	}
 
-    /**
-     * Provide the progress of computation, in percent, or -1 if
-     * that feature is not implemented.
-     * @return -1 if not implemented, or an integer between 0 and 100.
-     */
-    public int getProgress()
-    {
-        return percent;
-    }
-    
-    /**
-     * @param args
-     */
-    public static void main(String[] args) {
-        // TODO Auto-generated method stub
+	@Override
+	protected void initialiseComp() {
 
-    }
+		String timelineDir = db.getProp(db.VOCALIZATIONSDIR) + File.separator + "files";
+		if (!(new File(timelineDir)).exists()) {
+			System.out.println("vocalizations/files directory does not exist; ");
+			if (!(new File(timelineDir)).mkdirs()) {
+				throw new Error("Could not create vocalizations/files");
+			}
+			System.out.println("Created successfully.\n");
+		}
+
+		try {
+			String basenameFile = db.getProp(db.VOCALIZATIONSDIR) + File.separator + "basenames.lst";
+			if ((new File(basenameFile)).exists()) {
+				System.out.println("Loading basenames of vocalizations from '" + basenameFile + "' list...");
+				bnlVocalizations = new BasenameList(basenameFile);
+				System.out.println("Found " + bnlVocalizations.getLength() + " vocalizations in basename list");
+			} else {
+				String vocalWavDir = db.getProp(db.VOCALIZATIONSDIR) + File.separator + "wav";
+				System.out.println("Loading basenames of vocalizations from '" + vocalWavDir + "' directory...");
+				bnlVocalizations = new BasenameList(vocalWavDir, ".wav");
+				System.out.println("Found " + bnlVocalizations.getLength() + " vocalizations in " + vocalWavDir + " directory");
+			}
+		} catch (IOException e) {
+			e.printStackTrace();
+		}
+	}
+
+	public SortedMap getDefaultProps(DatabaseLayout db) {
+		this.db = db;
+		if (props == null) {
+			props = new TreeMap<String, String>();
+			props.put(WAVEDIR, db.getProp(db.VOCALIZATIONSDIR) + File.separator + "wav");
+			props.put(UNITFILE, db.getProp(db.VOCALIZATIONSDIR) + File.separator + "files" + File.separator
+					+ "vocalization_units" + db.getProp(db.MARYEXT));
+			props.put(POLYORDER, "3");
+			props.put(ISEXTERNALF0, "true");
+			props.put(EXTERNALF0FORMAT, "sptk");
+			props.put(EXTERNALEXT, ".lf0");
+			props.put(PITCHDIR, db.getProp(db.VOCALIZATIONSDIR) + File.separator + "lf0");
+			props.put(SKIPSIZE, "0.005");
+			props.put(WINDOWSIZE, "0.005");
+			props.put(F0TIMELINE, db.getProp(db.VOCALIZATIONSDIR) + File.separator + "files" + File.separator
+					+ "vocalization_intonation" + db.getProp(db.MARYEXT));
+			props.put(F0FEATDEF, db.getProp(db.VOCALIZATIONSDIR) + File.separator + "features" + File.separator
+					+ "vocalization_f0_feature_definition.txt");
+		}
+		return props;
+	}
+
+	protected void setupHelp() {
+		props2Help = new TreeMap();
+
+	}
+
+	/**
+	 * Reads and concatenates a list of waveforms into one single timeline file.
+	 * 
+	 * @throws IOException
+	 */
+	@Override
+	public boolean compute() throws IOException, MaryConfigurationException {
+
+		listenerUnits = new VocalizationUnitFileReader(getProp(UNITFILE));
+
+		// write features into timeline file
+		DataOutputStream out = new DataOutputStream(new BufferedOutputStream(new FileOutputStream(new File(getProp(F0TIMELINE)))));
+		writeHeaderTo(out);
+		writeUnitFeaturesTo(out);
+		out.close();
+
+		VocalizationIntonationReader tester = new VocalizationIntonationReader(getProp(F0TIMELINE));
+		int unitsOnDisk = tester.getNumberOfUnits();
+		if (unitsOnDisk == listenerUnits.getNumberOfUnits()) {
+			System.out.println("Can read right number of units");
+			return true;
+		} else {
+			System.out.println("Read wrong number of units: " + unitsOnDisk);
+			return false;
+		}
+	}
+
+	/**
+	 * 
+	 * @param out
+	 * @throws IOException
+	 */
+	protected void writeUnitFeaturesTo(DataOutput out) throws IOException {
+
+		int numUnits = listenerUnits.getNumberOfUnits();
+		float windowSize = new Float(getProp(WINDOWSIZE)).floatValue();
+		float skipSize = new Float(getProp(SKIPSIZE)).floatValue();
+
+		out.writeFloat(windowSize);
+		out.writeFloat(skipSize);
+		out.writeInt(numUnits);
+
+		for (int i = 0; i < bnlVocalizations.getLength(); i++) {
+
+			double[] f0Array = null;
+
+			try {
+				f0Array = getVocalizationF0(bnlVocalizations.getName(i), false);
+			} catch (UnsupportedAudioFileException e) {
+				e.printStackTrace();
+			}
+
+			// write coeffs followed by its order
+			double[] coeffs = getPolynomialCoeffs(f0Array);
+			if (coeffs == null) {
+				out.writeInt(0);
+			} else {
+				out.writeInt(coeffs.length);
+				for (int j = 0; j < coeffs.length; j++) {
+					out.writeFloat((float) coeffs[j]);
+				}
+			}
+
+			// write f0 Array followed by f0 contour array size
+			if (f0Array == null) {
+				out.writeInt(0);
+			} else {
+				out.writeInt(f0Array.length);
+				for (int j = 0; j < f0Array.length; j++) {
+					out.writeFloat((float) f0Array[j]);
+				}
+			}
+		}
+	}
+
+	/**
+	 * get f0 contour of vocalization f0
+	 * 
+	 * @param baseName
+	 * @return
+	 * @throws UnsupportedAudioFileException
+	 * @throws IOException
+	 */
+	private double[] getVocalizationF0(String baseName, boolean doInterpolate) throws UnsupportedAudioFileException, IOException {
+
+		double[] f0Array = null;
+
+		if ("true".equals(getProp(ISEXTERNALF0))) {
+
+			String externalFormat = getProp(EXTERNALF0FORMAT);
+			String externalExt = getProp(EXTERNALEXT);
+			System.out.println("Loading f0 contour from file : " + getProp(PITCHDIR) + File.separator + baseName + externalExt);
+			if ("sptk".equals(externalFormat)) {
+				String fileName = getProp(PITCHDIR) + File.separator + baseName + externalExt;
+				SPTKPitchReaderWriter sprw = new SPTKPitchReaderWriter(fileName);
+				f0Array = sprw.getF0Contour();
+			} else if ("ptc".equals(externalFormat)) {
+				String fileName = getProp(PITCHDIR) + File.separator + baseName + externalExt;
+				PitchReaderWriter sprw = new PitchReaderWriter(fileName);
+				f0Array = sprw.contour;
+			}
+		} else {
+			PitchFileHeader params = new PitchFileHeader();
+			F0TrackerAutocorrelationHeuristic tracker = new F0TrackerAutocorrelationHeuristic(params);
+			String waveFile = db.getProp(db.VOCALIZATIONSDIR) + File.separator + "wav" + baseName + db.getProp(db.WAVEXT);
+			System.out.println("Computing f0 contour from wave file: " + waveFile);
+			AudioInputStream inputAudio = AudioSystem.getAudioInputStream(new File(waveFile));
+
+			// Enforce PCM_SIGNED encoding
+			if (!inputAudio.getFormat().getEncoding().equals(AudioFormat.Encoding.PCM_SIGNED)) {
+				inputAudio = AudioSystem.getAudioInputStream(AudioFormat.Encoding.PCM_SIGNED, inputAudio);
+			}
+
+			int audioSampleRate = (int) inputAudio.getFormat().getSampleRate();
+			AudioDoubleDataSource signal = new AudioDoubleDataSource(inputAudio);
+			double[] sentenceAudio = signal.getAllData();
+			tracker.pitchAnalyze(new BufferedDoubleDataSource(sentenceAudio));
+			// double frameShiftTime = tracker.getSkipSizeInSeconds();
+			f0Array = tracker.getF0Contour();
+		}
+
+		if (doInterpolate) {
+			return interpolateF0Array(f0Array);
+		}
+
+		return f0Array;
+	}
+
+	/**
+	 * to get polynomial coeffs of f0 contour
+	 * 
+	 * @param f0Array
+	 * @return
+	 */
+	private double[] getPolynomialCoeffs(double[] f0Array) {
+
+		if (f0Array == null) {
+			return null;
+		}
+
+		f0Array = cutStartEndUnvoicedSegments(f0Array);
+		double[] f0AndInterpolate = interpolateF0Array(f0Array);
+		int polynomialOrder = (new Integer(getProp(POLYORDER))).intValue();
+		double[] coeffs = Polynomial.fitPolynomial(f0AndInterpolate, polynomialOrder);
+		return coeffs;
+	}
+
+	/**
+	 * to interpolate F0 contour values
+	 * 
+	 * @param f0Array
+	 * @return
+	 */
+	private double[] interpolateF0Array(double[] f0Array) {
+
+		if (f0Array == null) {
+			return null;
+		}
+
+		for (int j = 0; j < f0Array.length; j++) {
+			if (f0Array[j] == 0) {
+				f0Array[j] = Double.NaN;
+			}
+		}
+		if (f0Array.length >= 3) {
+			f0Array = SignalProcUtils.medianFilter(f0Array, 5);
+		}
+
+		double[] f0AndInterpol;
+		double[] interpol = new double[f0Array.length];
+		Arrays.fill(interpol, Double.NaN);
+		f0AndInterpol = new double[f0Array.length];
+		int iLastValid = -1;
+		for (int j = 0; j < f0Array.length; j++) {
+			if (!Double.isNaN(f0Array[j])) { // a valid value
+				if (iLastValid == j - 1) {
+					// no need to interpolate
+					f0AndInterpol[j] = f0Array[j];
+				} else {
+					// need to interpolate
+					double prevF0;
+					if (iLastValid < 0) { // we don't have a previous value -- use current one
+						prevF0 = f0Array[j];
+					} else {
+						prevF0 = f0Array[iLastValid];
+					}
+					double delta = (f0Array[j] - prevF0) / (j - iLastValid);
+					double f0 = prevF0;
+					for (int k = iLastValid + 1; k < j; k++) {
+						f0 += delta;
+						interpol[k] = f0;
+						f0AndInterpol[k] = f0;
+					}
+				}
+				iLastValid = j;
+			}
+		}
+
+		double[] f0AndInterpolate = combineF0andInterpolate(f0Array, interpol);
+		return f0AndInterpolate;
+	}
+
+	/**
+	 * cut begin-end unvoiced segments
+	 * 
+	 * @param array
+	 * @return
+	 */
+	private double[] cutStartEndUnvoicedSegments(double[] array) {
+
+		if (array == null)
+			return null;
+
+		int startIndex = 0;
+		int endIndex = array.length;
+
+		// find start index
+		for (int i = 0; i < array.length; i++) {
+			if (array[i] != 0) {
+				startIndex = i;
+				break;
+			}
+		}
+
+		// find end index
+		for (int i = (array.length - 1); i > 0; i--) {
+			if (array[i] != 0) {
+				endIndex = i;
+				break;
+			}
+		}
+
+		int newArraySize = endIndex - startIndex;
+
+		double[] newArray = new double[newArraySize];
+		System.arraycopy(array, startIndex, newArray, 0, newArraySize);
+
+		/*
+		 * for ( int i=0; i<newArray.length; i++ ) { System.out.println(newArray[i]); }
+		 * System.out.println("Resized from "+array.length+" to "+newArraySize);
+		 */
+		return newArray;
+	}
+
+	/**
+	 * interpolate f0
+	 * 
+	 * @param f0Array
+	 * @param interpol
+	 * @return
+	 */
+	private double[] combineF0andInterpolate(double[] f0Array, double[] interpol) {
+
+		double[] f0AndInterpolate = new double[f0Array.length];
+		Arrays.fill(f0AndInterpolate, Double.NaN);
+		for (int i = 0; i < f0Array.length; i++) {
+			if (!Double.isNaN(f0Array[i])) {
+				f0AndInterpolate[i] = f0Array[i];
+			} else if (!Double.isNaN(interpol[i])) {
+				f0AndInterpolate[i] = interpol[i];
+			}
+			// System.out.println(f0Array[i]+" "+interpol[i]+" "+f0AndInterpolate[i]);
+		}
+		return f0AndInterpolate;
+	}
+
+	/**
+	 * Write the header of this feature file to the given DataOutput
+	 * 
+	 * @param out
+	 * @throws IOException
+	 */
+	protected void writeHeaderTo(DataOutput out) throws IOException {
+		new MaryHeader(MaryHeader.LISTENERFEATS).writeTo(out);
+	}
+
+	/**
+	 * Provide the progress of computation, in percent, or -1 if that feature is not implemented.
+	 * 
+	 * @return -1 if not implemented, or an integer between 0 and 100.
+	 */
+	public int getProgress() {
+		return percent;
+	}
+
+	/**
+	 * @param args
+	 */
+	public static void main(String[] args) {
+		// TODO Auto-generated method stub
+
+	}
 
 }

marytts-builder/src/main/java/marytts/tools/voiceimport/vocalizations/VocalizationPitchmarker.java

@@ -33,131 +33,128 @@ import marytts.util.io.BasenameList;
 
 public class VocalizationPitchmarker extends PraatPitchmarker {
 
-    String vocalizationsDir;
-    BasenameList bnlVocalizations;
-    
-    
-    public String getName(){
-        return "VocalizationPitchmarker";
-    }
-    
-    @Override
-    protected void initialiseComp()
-    {
-        tmpScript = db.getProp(db.TEMPDIR)+"script.praat";
-        
-        if (!(new File(getProp(PRAATPMDIR))).exists()) {
-        
-            System.out.println("vocalizations/pm directory does not exist; ");
-            if (!(new File(getProp(PRAATPMDIR))).mkdirs()) {
-                throw new Error("Could not create vocalizations/pm");
-            }
-            System.out.println("Created successfully.\n");
-            
-        }
-        
-        try {
-            String basenameFile = db.getProp(db.VOCALIZATIONSDIR)+File.separator+"basenames.lst";
-            if ( (new File(basenameFile)).exists() ) {
-                System.out.println("Loading basenames of vocalizations from '"+basenameFile+"' list...");
-                bnlVocalizations = new BasenameList(basenameFile);
-                System.out.println("Found "+bnlVocalizations.getLength()+ " vocalizations in basename list");
-            }
-            else {
-                String vocalWavDir = db.getProp(db.VOCALIZATIONSDIR)+File.separator+"wav";
-                System.out.println("Loading basenames of vocalizations from '"+vocalWavDir+"' directory...");
-                bnlVocalizations = new BasenameList(vocalWavDir, ".wav");
-                System.out.println("Found "+bnlVocalizations.getLength()+ " vocalizations in "+ vocalWavDir + " directory");
-            }
-        } catch (IOException e) {
-            e.printStackTrace();
-        }
-    }
-    
-    public SortedMap getDefaultProps(DatabaseLayout db){
-        this.db = db;
-       if (props == null){
-           props = new TreeMap();
-           props.put(COMMAND,"praat");   
-           if (db.getProp(db.GENDER).equals("female")){
-               props.put(MINPITCH,"100");
-               props.put(MAXPITCH,"500");
-           } else {
-               props.put(MINPITCH,"75");
-               props.put(MAXPITCH,"300");
-           }
-           props.put(WAVEDIR,db.getProp(db.VOCALIZATIONSDIR)+File.separator+"wav");
-           props.put(PRAATPMDIR,db.getProp(db.VOCALIZATIONSDIR)+File.separator+"pm");
-           //vocalizationsDir = db.getProp(db.ROOTDIR)+File.separator+"vocalizations";
-       }
-       return props;
-    }
-    
-    /**
-     * The standard compute() method of the VoiceImportComponent interface.
-     */
-    public boolean compute() throws IOException {
-        
-        String[] baseNameArray = bnlVocalizations.getListAsArray();
-        System.out.println( "Computing pitchmarks for " + baseNameArray.length + " utterances." );
+	String vocalizationsDir;
+	BasenameList bnlVocalizations;
 
-        /* Ensure the existence of the target pitchmark directory */
-        File dir = new File(getProp(PRAATPMDIR));
-        if (!dir.exists()) {
-            System.out.println( "Creating the directory [" + getProp(PRAATPMDIR) + "]." );
-            dir.mkdirs();
-        }
-        
-        // script.praat is provided as template. Perhaps it could be used instead of hardcoding the following:
-        File script = new File(tmpScript);
-        
-        if (script.exists()) script.delete();
-        PrintWriter toScript = new PrintWriter(new FileWriter(script));
-        // use Praat form to provide ARGV (NOTE: these must be explicitly given during call to Praat!)
-        toScript.println("form Provide arguments");
-        toScript.println("  sentence wavFile input.wav");
-        toScript.println("  sentence pointpFile output.PointProcess");
-        toScript.println("  real minPitch 75");
-        toScript.println("  real maxPitch 600");
-        toScript.println("endform");
-        toScript.println("Read from file... 'wavFile$'");
-        // Remove DC offset, if present:
-        toScript.println("Subtract mean");
-        // First, low-pass filter the speech signal to make it more robust against noise
-        // (i.e., mixed noise+periodicity regions treated more likely as periodic)
-        toScript.println("sound = Filter (pass Hann band)... 0 1000 100");
-        // Then determine pitch curve:
-        toScript.println("pitch = To Pitch... 0 minPitch maxPitch");
-        // Get some debug info:
-        toScript.println("min_f0 = Get minimum... 0 0 Hertz Parabolic");
-        toScript.println("max_f0 = Get maximum... 0 0 Hertz Parabolic");
-        // And convert to pitch marks:
-        toScript.println("plus sound");
-        toScript.println("To PointProcess (cc)");
-        // Fill in 100 Hz pseudo-pitchmarks in unvoiced regions:
-        toScript.println("Voice... 0.01 0.02000000001");
-        toScript.println("Write to short text file... 'pointpFile$'");
-        toScript.println("lastSlash = rindex(wavFile$, \"/\")");
-        toScript.println("baseName$ = right$(wavFile$, length(wavFile$) - lastSlash) - \".wav\"");
-        toScript.println("printline 'baseName$'   f0 range: 'min_f0:0' - 'max_f0:0' Hz");
-        toScript.close();
-        
-        System.out.println("Running Praat as: "+getProp(COMMAND)+" "+tmpScript);
-        for ( int i = 0; i < baseNameArray.length; i++ ) {
-            percent = 100*i/baseNameArray.length;
-            praatPitchmarks(baseNameArray[i]);
-        }
+	public String getName() {
+		return "VocalizationPitchmarker";
+	}
 
-        return true;
-    }
-    
-    
-    /**
-     * @param args
-     */
-    public static void main(String[] args) {
-        // TODO Auto-generated method stub
+	@Override
+	protected void initialiseComp() {
+		tmpScript = db.getProp(db.TEMPDIR) + "script.praat";
 
-    }
+		if (!(new File(getProp(PRAATPMDIR))).exists()) {
+
+			System.out.println("vocalizations/pm directory does not exist; ");
+			if (!(new File(getProp(PRAATPMDIR))).mkdirs()) {
+				throw new Error("Could not create vocalizations/pm");
+			}
+			System.out.println("Created successfully.\n");
+
+		}
+
+		try {
+			String basenameFile = db.getProp(db.VOCALIZATIONSDIR) + File.separator + "basenames.lst";
+			if ((new File(basenameFile)).exists()) {
+				System.out.println("Loading basenames of vocalizations from '" + basenameFile + "' list...");
+				bnlVocalizations = new BasenameList(basenameFile);
+				System.out.println("Found " + bnlVocalizations.getLength() + " vocalizations in basename list");
+			} else {
+				String vocalWavDir = db.getProp(db.VOCALIZATIONSDIR) + File.separator + "wav";
+				System.out.println("Loading basenames of vocalizations from '" + vocalWavDir + "' directory...");
+				bnlVocalizations = new BasenameList(vocalWavDir, ".wav");
+				System.out.println("Found " + bnlVocalizations.getLength() + " vocalizations in " + vocalWavDir + " directory");
+			}
+		} catch (IOException e) {
+			e.printStackTrace();
+		}
+	}
+
+	public SortedMap getDefaultProps(DatabaseLayout db) {
+		this.db = db;
+		if (props == null) {
+			props = new TreeMap();
+			props.put(COMMAND, "praat");
+			if (db.getProp(db.GENDER).equals("female")) {
+				props.put(MINPITCH, "100");
+				props.put(MAXPITCH, "500");
+			} else {
+				props.put(MINPITCH, "75");
+				props.put(MAXPITCH, "300");
+			}
+			props.put(WAVEDIR, db.getProp(db.VOCALIZATIONSDIR) + File.separator + "wav");
+			props.put(PRAATPMDIR, db.getProp(db.VOCALIZATIONSDIR) + File.separator + "pm");
+			// vocalizationsDir = db.getProp(db.ROOTDIR)+File.separator+"vocalizations";
+		}
+		return props;
+	}
+
+	/**
+	 * The standard compute() method of the VoiceImportComponent interface.
+	 */
+	public boolean compute() throws IOException {
+
+		String[] baseNameArray = bnlVocalizations.getListAsArray();
+		System.out.println("Computing pitchmarks for " + baseNameArray.length + " utterances.");
+
+		/* Ensure the existence of the target pitchmark directory */
+		File dir = new File(getProp(PRAATPMDIR));
+		if (!dir.exists()) {
+			System.out.println("Creating the directory [" + getProp(PRAATPMDIR) + "].");
+			dir.mkdirs();
+		}
+
+		// script.praat is provided as template. Perhaps it could be used instead of hardcoding the following:
+		File script = new File(tmpScript);
+
+		if (script.exists())
+			script.delete();
+		PrintWriter toScript = new PrintWriter(new FileWriter(script));
+		// use Praat form to provide ARGV (NOTE: these must be explicitly given during call to Praat!)
+		toScript.println("form Provide arguments");
+		toScript.println("  sentence wavFile input.wav");
+		toScript.println("  sentence pointpFile output.PointProcess");
+		toScript.println("  real minPitch 75");
+		toScript.println("  real maxPitch 600");
+		toScript.println("endform");
+		toScript.println("Read from file... 'wavFile$'");
+		// Remove DC offset, if present:
+		toScript.println("Subtract mean");
+		// First, low-pass filter the speech signal to make it more robust against noise
+		// (i.e., mixed noise+periodicity regions treated more likely as periodic)
+		toScript.println("sound = Filter (pass Hann band)... 0 1000 100");
+		// Then determine pitch curve:
+		toScript.println("pitch = To Pitch... 0 minPitch maxPitch");
+		// Get some debug info:
+		toScript.println("min_f0 = Get minimum... 0 0 Hertz Parabolic");
+		toScript.println("max_f0 = Get maximum... 0 0 Hertz Parabolic");
+		// And convert to pitch marks:
+		toScript.println("plus sound");
+		toScript.println("To PointProcess (cc)");
+		// Fill in 100 Hz pseudo-pitchmarks in unvoiced regions:
+		toScript.println("Voice... 0.01 0.02000000001");
+		toScript.println("Write to short text file... 'pointpFile$'");
+		toScript.println("lastSlash = rindex(wavFile$, \"/\")");
+		toScript.println("baseName$ = right$(wavFile$, length(wavFile$) - lastSlash) - \".wav\"");
+		toScript.println("printline 'baseName$'   f0 range: 'min_f0:0' - 'max_f0:0' Hz");
+		toScript.close();
+
+		System.out.println("Running Praat as: " + getProp(COMMAND) + " " + tmpScript);
+		for (int i = 0; i < baseNameArray.length; i++) {
+			percent = 100 * i / baseNameArray.length;
+			praatPitchmarks(baseNameArray[i]);
+		}
+
+		return true;
+	}
+
+	/**
+	 * @param args
+	 */
+	public static void main(String[] args) {
+		// TODO Auto-generated method stub
+
+	}
 
 }

marytts-builder/src/main/java/marytts/tools/voiceimport/vocalizations/VocalizationTimelineMaker.java

@@ -38,205 +38,196 @@ import marytts.util.io.BasenameList;
 
 public class VocalizationTimelineMaker extends VoiceImportComponent {
 
-    String vocalizationsDir;
-    BasenameList bnlVocalizations;
-    
-    protected DatabaseLayout db = null;
-    protected int percent = 0;
-    public final String WAVETIMELINE = "VocalizationTimelineMaker.waveTimeline";
-    public final String WAVEDIR = "VocalizationTimelineMaker.inputWaveDir";
-    public final String PMARKDIR = "VocalizationTimelineMaker.pitchmarkDir";
-    
-    public final String PMDIR = "db.pmDir";
-    public final String PMEXT = "db.pmExtension";
+	String vocalizationsDir;
+	BasenameList bnlVocalizations;
 
-    
-    public String getName(){
-        return "VocalizationTimelineMaker";
-    }
-    
-    @Override
-    protected void initialiseComp()
-    {
-        String timelineDir = db.getProp(db.VOCALIZATIONSDIR) + File.separator + "files";
-        if (!(new File(timelineDir)).exists()) {
-        
-            System.out.println("vocalizations/files directory does not exist; ");
-            if (!(new File(timelineDir)).mkdirs()) {
-                throw new Error("Could not create vocalizations/files");
-            }
-            System.out.println("Created successfully.\n");
-            
-        }
-        
-        try {
-            String basenameFile = db.getProp(db.VOCALIZATIONSDIR)+File.separator+"basenames.lst";
-            if ( (new File(basenameFile)).exists() ) {
-                System.out.println("Loading basenames of vocalizations from '"+basenameFile+"' list...");
-                bnlVocalizations = new BasenameList(basenameFile);
-                System.out.println("Found "+bnlVocalizations.getLength()+ " vocalizations in basename list");
-            }
-            else {
-                String vocalWavDir = db.getProp(db.VOCALIZATIONSDIR)+File.separator+"wav";
-                System.out.println("Loading basenames of vocalizations from '"+vocalWavDir+"' directory...");
-                bnlVocalizations = new BasenameList(vocalWavDir, ".wav");
-                System.out.println("Found "+bnlVocalizations.getLength()+ " vocalizations in "+ vocalWavDir + " directory");
-            }
-        } catch (IOException e) {
-            e.printStackTrace();
-        }
-    }
-    
-    public SortedMap getDefaultProps(DatabaseLayout db){
-        this.db = db;
-       if (props == null){
-           props = new TreeMap();
-           props.put(WAVEDIR,db.getProp(db.VOCALIZATIONSDIR)+File.separator+"wav");
-           props.put(PMARKDIR,db.getProp(db.VOCALIZATIONSDIR)+File.separator+"pm");
-           props.put(WAVETIMELINE, db.getProp(db.VOCALIZATIONSDIR) + File.separator
-                   + "files" + File.separator + "vocalization_wave_timeline"+db.getProp(db.MARYEXT));
-           //vocalizationsDir = db.getProp(db.ROOTDIR)+File.separator+"vocalizations";
-       }
-       return props;
-    }
-    
-    protected void setupHelp(){
-        props2Help = new TreeMap();
-        props2Help.put(WAVETIMELINE,"file containing all wave files. Will be created by this module");
-    }
-    
-    /**
-     *  Reads and concatenates a list of waveforms into one single timeline file.
-     *
-     */
-    public boolean compute()
-    {
-        System.out.println("---- Making a pitch synchronous waveform timeline\n\n");
-        
-        /* Export the basename list into an array of strings */
-        String[] baseNameArray = bnlVocalizations.getListAsArray();
-        System.out.println("Processing [" + baseNameArray.length + "] utterances.\n");
-        
-       
-        
-        try{
-            /* 1) Determine the reference sampling rate as being the sample rate of the first encountered
-             *    wav file */
-            WavReader wav = new WavReader( getProp(WAVEDIR) 
-                    + baseNameArray[0] + db.getProp(db.WAVEXT));
-            int globSampleRate = wav.getSampleRate();
-            System.out.println("---- Detected a global sample rate of: [" + globSampleRate + "] Hz." );
-            
-            System.out.println("---- Folding the wav files according to the pitchmarks..." );
-            
-            /* 2) Open the destination timeline file */
-            
-            /* Make the file name */
-            String waveTimelineName = getProp(WAVETIMELINE);
-            System.out.println( "Will create the waveform timeline in file [" + waveTimelineName + "]." );
-            
-            /* Processing header: */
-            String processingHeader = "\n";
-            
-            /* Instantiate the TimelineWriter: */
-            TimelineWriter waveTimeline = new TimelineWriter( waveTimelineName, processingHeader, globSampleRate, 0.1 );
-            
-            /* 3) Write the datagrams and feed the index */
-            
-            float totalDuration = 0.0f;  // Accumulator for the total timeline duration
-            long totalTime = 0l;
-            int numDatagrams = 0;
-            
-            /* For each EST track file: */
-            ESTTrackReader pmFile = null;
-            for ( int i = 0; i < baseNameArray.length; i++ ) {
-                percent = 100*i/baseNameArray.length;
+	protected DatabaseLayout db = null;
+	protected int percent = 0;
+	public final String WAVETIMELINE = "VocalizationTimelineMaker.waveTimeline";
+	public final String WAVEDIR = "VocalizationTimelineMaker.inputWaveDir";
+	public final String PMARKDIR = "VocalizationTimelineMaker.pitchmarkDir";
 
-                /* - open+load */
-                System.out.println( baseNameArray[i] );
-                pmFile = new ESTTrackReader( getProp(PMARKDIR)
-                                + baseNameArray[i] + db.getProp(PMEXT));
-                totalDuration += pmFile.getTimeSpan();
-                wav = new WavReader( getProp(WAVEDIR) + baseNameArray[i] + db.getProp(db.WAVEXT) );
-                short[] wave = wav.getSamples();
-                /* - Reset the frame locations in the local file */
-                int frameStart = 0;
-                int frameEnd = 0;
-                int duration = 0;
-                long localTime = 0l;
-                /* - For each frame in the WAV file: */
-                for ( int f = 0; f < pmFile.getNumFrames(); f++ ) {
-                    
-                    /* Locate the corresponding segment in the wave file */
-                    frameStart = frameEnd;
-                    frameEnd = (int)( (double)pmFile.getTime( f ) * (double)(globSampleRate) );
-                    assert frameEnd <= wave.length : "Frame ends after end of wave data: " + frameEnd + " > " + wave.length;
-                    
-                    duration = frameEnd - frameStart;
-                    ByteArrayOutputStream buff =  new ByteArrayOutputStream(2*duration);
-                    DataOutputStream subWave = new DataOutputStream( buff );
-                    for (int k = 0; k < duration; k++) 
-                    {
-                        subWave.writeShort( wave[frameStart+k] );
-                    }
-                    
-                    //Handle the case when the last pitch marks falls beyond the end of the signal
-                    
-                    
-                    /* Feed the datagram to the timeline */
-                    waveTimeline.feed( new Datagram(duration, buff.toByteArray()), globSampleRate );
-                    totalTime += duration;
-                    localTime += duration;
-                    numDatagrams++;
-                }
-                // System.out.println( baseNameArray[i] + " -> pm file says [" + localTime + "] samples, wav file says ["+ wav.getNumSamples() + "] samples." );
-            }
-            waveTimeline.close();
-            
-            System.out.println("---- Done." );
-            
-            /* 7) Print some stats and close the file */
-            System.out.println( "---- Waveform timeline result:");
-            System.out.println( "Number of files scanned: " + baseNameArray.length );
-            System.out.println( "Total speech duration: [" + totalTime + "] samples / [" + ((float)(totalTime) / (float)(globSampleRate)) + "] seconds." );
-            System.out.println( "(Speech duration approximated from EST Track float times: [" + totalDuration + "] seconds.)" );
-            System.out.println( "Number of frames: [" + numDatagrams + "]." );
-            System.out.println( "Size of the index: [" + waveTimeline.getIndex().getNumIdx() + "] ("
-                    + (waveTimeline.getIndex().getNumIdx() * 16) + " bytes, i.e. "
-                    + new DecimalFormat("#.##").format((double)(waveTimeline.getIndex().getNumIdx()) * 16.0 / 1048576.0) + " megs).");
-            System.out.println( "---- Waveform timeline done.");
-            
-        }
-        catch ( SecurityException e ) {
-            System.err.println( "Error: you don't have write access to the target database directory." );
-            e.printStackTrace();
-            return false;
-        }
-        catch (Exception e) {
-            e.printStackTrace();
-            System.err.println(e);
-            return false;
-        }
-        
-        return( true );
-    }    
-    
-    /**
-     * Provide the progress of computation, in percent, or -1 if
-     * that feature is not implemented.
-     * @return -1 if not implemented, or an integer between 0 and 100.
-     */
-    public int getProgress()
-    {
-        return percent;
-    }
-    
-    /**
-     * @param args
-     */
-    public static void main(String[] args) {
-        // TODO Auto-generated method stub
+	public final String PMDIR = "db.pmDir";
+	public final String PMEXT = "db.pmExtension";
 
-    }
+	public String getName() {
+		return "VocalizationTimelineMaker";
+	}
+
+	@Override
+	protected void initialiseComp() {
+		String timelineDir = db.getProp(db.VOCALIZATIONSDIR) + File.separator + "files";
+		if (!(new File(timelineDir)).exists()) {
+
+			System.out.println("vocalizations/files directory does not exist; ");
+			if (!(new File(timelineDir)).mkdirs()) {
+				throw new Error("Could not create vocalizations/files");
+			}
+			System.out.println("Created successfully.\n");
+
+		}
+
+		try {
+			String basenameFile = db.getProp(db.VOCALIZATIONSDIR) + File.separator + "basenames.lst";
+			if ((new File(basenameFile)).exists()) {
+				System.out.println("Loading basenames of vocalizations from '" + basenameFile + "' list...");
+				bnlVocalizations = new BasenameList(basenameFile);
+				System.out.println("Found " + bnlVocalizations.getLength() + " vocalizations in basename list");
+			} else {
+				String vocalWavDir = db.getProp(db.VOCALIZATIONSDIR) + File.separator + "wav";
+				System.out.println("Loading basenames of vocalizations from '" + vocalWavDir + "' directory...");
+				bnlVocalizations = new BasenameList(vocalWavDir, ".wav");
+				System.out.println("Found " + bnlVocalizations.getLength() + " vocalizations in " + vocalWavDir + " directory");
+			}
+		} catch (IOException e) {
+			e.printStackTrace();
+		}
+	}
+
+	public SortedMap getDefaultProps(DatabaseLayout db) {
+		this.db = db;
+		if (props == null) {
+			props = new TreeMap();
+			props.put(WAVEDIR, db.getProp(db.VOCALIZATIONSDIR) + File.separator + "wav");
+			props.put(PMARKDIR, db.getProp(db.VOCALIZATIONSDIR) + File.separator + "pm");
+			props.put(WAVETIMELINE, db.getProp(db.VOCALIZATIONSDIR) + File.separator + "files" + File.separator
+					+ "vocalization_wave_timeline" + db.getProp(db.MARYEXT));
+			// vocalizationsDir = db.getProp(db.ROOTDIR)+File.separator+"vocalizations";
+		}
+		return props;
+	}
+
+	protected void setupHelp() {
+		props2Help = new TreeMap();
+		props2Help.put(WAVETIMELINE, "file containing all wave files. Will be created by this module");
+	}
+
+	/**
+	 * Reads and concatenates a list of waveforms into one single timeline file.
+	 *
+	 */
+	public boolean compute() {
+		System.out.println("---- Making a pitch synchronous waveform timeline\n\n");
+
+		/* Export the basename list into an array of strings */
+		String[] baseNameArray = bnlVocalizations.getListAsArray();
+		System.out.println("Processing [" + baseNameArray.length + "] utterances.\n");
+
+		try {
+			/*
+			 * 1) Determine the reference sampling rate as being the sample rate of the first encountered wav file
+			 */
+			WavReader wav = new WavReader(getProp(WAVEDIR) + baseNameArray[0] + db.getProp(db.WAVEXT));
+			int globSampleRate = wav.getSampleRate();
+			System.out.println("---- Detected a global sample rate of: [" + globSampleRate + "] Hz.");
+
+			System.out.println("---- Folding the wav files according to the pitchmarks...");
+
+			/* 2) Open the destination timeline file */
+
+			/* Make the file name */
+			String waveTimelineName = getProp(WAVETIMELINE);
+			System.out.println("Will create the waveform timeline in file [" + waveTimelineName + "].");
+
+			/* Processing header: */
+			String processingHeader = "\n";
+
+			/* Instantiate the TimelineWriter: */
+			TimelineWriter waveTimeline = new TimelineWriter(waveTimelineName, processingHeader, globSampleRate, 0.1);
+
+			/* 3) Write the datagrams and feed the index */
+
+			float totalDuration = 0.0f; // Accumulator for the total timeline duration
+			long totalTime = 0l;
+			int numDatagrams = 0;
+
+			/* For each EST track file: */
+			ESTTrackReader pmFile = null;
+			for (int i = 0; i < baseNameArray.length; i++) {
+				percent = 100 * i / baseNameArray.length;
+
+				/* - open+load */
+				System.out.println(baseNameArray[i]);
+				pmFile = new ESTTrackReader(getProp(PMARKDIR) + baseNameArray[i] + db.getProp(PMEXT));
+				totalDuration += pmFile.getTimeSpan();
+				wav = new WavReader(getProp(WAVEDIR) + baseNameArray[i] + db.getProp(db.WAVEXT));
+				short[] wave = wav.getSamples();
+				/* - Reset the frame locations in the local file */
+				int frameStart = 0;
+				int frameEnd = 0;
+				int duration = 0;
+				long localTime = 0l;
+				/* - For each frame in the WAV file: */
+				for (int f = 0; f < pmFile.getNumFrames(); f++) {
+
+					/* Locate the corresponding segment in the wave file */
+					frameStart = frameEnd;
+					frameEnd = (int) ((double) pmFile.getTime(f) * (double) (globSampleRate));
+					assert frameEnd <= wave.length : "Frame ends after end of wave data: " + frameEnd + " > " + wave.length;
+
+					duration = frameEnd - frameStart;
+					ByteArrayOutputStream buff = new ByteArrayOutputStream(2 * duration);
+					DataOutputStream subWave = new DataOutputStream(buff);
+					for (int k = 0; k < duration; k++) {
+						subWave.writeShort(wave[frameStart + k]);
+					}
+
+					// Handle the case when the last pitch marks falls beyond the end of the signal
+
+					/* Feed the datagram to the timeline */
+					waveTimeline.feed(new Datagram(duration, buff.toByteArray()), globSampleRate);
+					totalTime += duration;
+					localTime += duration;
+					numDatagrams++;
+				}
+				// System.out.println( baseNameArray[i] + " -> pm file says [" + localTime + "] samples, wav file says ["+
+				// wav.getNumSamples() + "] samples." );
+			}
+			waveTimeline.close();
+
+			System.out.println("---- Done.");
+
+			/* 7) Print some stats and close the file */
+			System.out.println("---- Waveform timeline result:");
+			System.out.println("Number of files scanned: " + baseNameArray.length);
+			System.out.println("Total speech duration: [" + totalTime + "] samples / ["
+					+ ((float) (totalTime) / (float) (globSampleRate)) + "] seconds.");
+			System.out.println("(Speech duration approximated from EST Track float times: [" + totalDuration + "] seconds.)");
+			System.out.println("Number of frames: [" + numDatagrams + "].");
+			System.out.println("Size of the index: [" + waveTimeline.getIndex().getNumIdx() + "] ("
+					+ (waveTimeline.getIndex().getNumIdx() * 16) + " bytes, i.e. "
+					+ new DecimalFormat("#.##").format((double) (waveTimeline.getIndex().getNumIdx()) * 16.0 / 1048576.0)
+					+ " megs).");
+			System.out.println("---- Waveform timeline done.");
+
+		} catch (SecurityException e) {
+			System.err.println("Error: you don't have write access to the target database directory.");
+			e.printStackTrace();
+			return false;
+		} catch (Exception e) {
+			e.printStackTrace();
+			System.err.println(e);
+			return false;
+		}
+
+		return (true);
+	}
+
+	/**
+	 * Provide the progress of computation, in percent, or -1 if that feature is not implemented.
+	 * 
+	 * @return -1 if not implemented, or an integer between 0 and 100.
+	 */
+	public int getProgress() {
+		return percent;
+	}
+
+	/**
+	 * @param args
+	 */
+	public static void main(String[] args) {
+		// TODO Auto-generated method stub
+
+	}
 
 }

marytts-builder/src/main/java/marytts/tools/voiceimport/vocalizations/VocalizationUnitfileWriter.java

@@ -44,281 +44,276 @@ import marytts.vocalizations.VocalizationUnitFileReader;
 
 /**
  * Back-channel unit writer
+ * 
  * @author sathish pammi
  *
  */
-public class VocalizationUnitfileWriter extends VoiceImportComponent
-{
-    
-    protected String vocalizationsDir;
-    protected BasenameList bnlVocalizations;
-    
-    //protected File maryDir;
-    protected String unitFileName;
-    protected File unitlabelDir;
-    protected int samplingRate;
-    //protected String pauseSymbol;
-    protected String unitlabelExt = ".lab";
-    
-    protected DatabaseLayout db = null;
-    protected int percent = 0;
-    //protected BasenameList bachChannelList;
-    
-    public String LABELDIR = "VocalizationUnitfileWriter.backchannelLabDir";
-    public String UNITFILE = "VocalizationUnitfileWriter.unitFile";
-    public final String PMARKDIR = "VocalizationUnitfileWriter.pitchmarkDir";
-    //public String BASELIST = "VocalizationUnitfileWriter.backchannelBaseNamesList";
+public class VocalizationUnitfileWriter extends VoiceImportComponent {
 
-    public final String PMDIR = "db.pmDir";
-    public final String PMEXT = "db.pmExtension";
+	protected String vocalizationsDir;
+	protected BasenameList bnlVocalizations;
 
-    public String getName(){
-        return "VocalizationUnitfileWriter";
-    }
-    
-    @Override
-    protected void initialiseComp()
-    {
-        //maryDir = new File(db.getProp(db.FILEDIR));
-        
-        samplingRate = Integer.parseInt(db.getProp(db.SAMPLINGRATE));
-        //pauseSymbol = System.getProperty("pause.symbol", "pau");
-    
-        unitFileName = getProp(UNITFILE);
-        unitlabelDir = new File(getProp(LABELDIR));
-        
-        String timelineDir = db.getProp(db.VOCALIZATIONSDIR) + File.separator + "files";
-        if (!(new File(timelineDir)).exists()) {
-        
-            System.out.println("vocalizations/files directory does not exist; ");
-            if (!(new File(timelineDir)).mkdirs()) {
-                throw new Error("Could not create vocalizations/files");
-            }
-            System.out.println("Created successfully.\n");
-            
-        }
-        
-        try {
-            String basenameFile = db.getProp(db.VOCALIZATIONSDIR)+File.separator+"basenames.lst";
-            if ( (new File(basenameFile)).exists() ) {
-                System.out.println("Loading basenames of vocalizations from '"+basenameFile+"' list...");
-                bnlVocalizations = new BasenameList(basenameFile);
-                System.out.println("Found "+bnlVocalizations.getLength()+ " vocalizations in basename list");
-            }
-            else {
-                String vocalWavDir = db.getProp(db.VOCALIZATIONSDIR)+File.separator+"wav";
-                System.out.println("Loading basenames of vocalizations from '"+vocalWavDir+"' directory...");
-                bnlVocalizations = new BasenameList(vocalWavDir, ".wav");
-                System.out.println("Found "+bnlVocalizations.getLength()+ " vocalizations in "+ vocalWavDir + " directory");
-            }
-        } catch (IOException e) {
-            e.printStackTrace();
-        }
-        
-    }
-    
-    public SortedMap getDefaultProps(DatabaseLayout db){
-       this.db = db;
-       if (props == null){
-           props = new TreeMap();
-           String vocalizationsrootDir = db.getProp(db.VOCALIZATIONSDIR);
-           props.put(LABELDIR, vocalizationsrootDir + File.separator
-                   +"lab"
-                   +System.getProperty("file.separator"));
-           props.put(UNITFILE, vocalizationsrootDir + File.separator + "files" + File.separator
-                   +"vocalization_units"+db.getProp(db.MARYEXT));           
-           props.put(PMARKDIR,db.getProp(db.VOCALIZATIONSDIR)+File.separator+"pm");
-           //props.put(BASELIST, "backchannel.lst");
-       }
-       return props;
-    }
-    
-    protected void setupHelp(){
-        props2Help = new TreeMap();
-        props2Help.put(LABELDIR, "directory containing the phone labels");
-        props2Help.put(UNITFILE, "file containing all phone units. Will be created by this module");           
-    }
-    
-    @Override
-    public boolean compute() throws IOException, MaryConfigurationException
-    {
-        if (!unitlabelDir.exists()){
-            System.out.print(LABELDIR+" "+getProp(LABELDIR)+" does not exist; ");
-            throw new Error("LABELDIR not found");
-        }  
-       
-        System.out.println("Back channel unitfile writer started...");
-        BackChannelUnits bcUnits = new BackChannelUnits(unitlabelDir.getAbsolutePath(),this.bnlVocalizations);
-        DataOutputStream out = new DataOutputStream(new BufferedOutputStream(new FileOutputStream(unitFileName)));
-        long posNumUnits = new MaryHeader(MaryHeader.LISTENERUNITS).writeTo(out);
-        int numberOfBCUnits = bcUnits.getNumberOfUnits();
-        out.writeInt(numberOfBCUnits); 
-        out.writeInt(samplingRate);
-        long globalStart = 0l; // time, given as sample position with samplingRate
-        for(int i=0;i<numberOfBCUnits;i++){
-            UnitLabel[] fileLabels =  bcUnits.getUnitLabels(i);
-            double unitTimeSpan  = bcUnits.getTimeSpan(i);
-            int localLabLength = fileLabels.length; 
-            out.writeInt(localLabLength);
-            for(int j=0;j<localLabLength;j++){
-                double startTime = fileLabels[j].startTime;
-                double endTime = fileLabels[j].endTime;
-                double duration = endTime - startTime;
-                long end = (long)( endTime * (double)(samplingRate) );
-                long start = (long)( startTime * (double)(samplingRate) );
-                out.writeLong( globalStart + start); out.writeInt((int) (end - start));
-                out.writeInt(fileLabels[j].unitName.toCharArray().length);
-                out.writeChars(fileLabels[j].unitName);
-            }
-            globalStart += ((long)( (double)( unitTimeSpan  ) * (double)(samplingRate) ));
-        }
+	// protected File maryDir;
+	protected String unitFileName;
+	protected File unitlabelDir;
+	protected int samplingRate;
+	// protected String pauseSymbol;
+	protected String unitlabelExt = ".lab";
 
-        out.close();
-        VocalizationUnitFileReader tester = new VocalizationUnitFileReader(unitFileName);
-        int unitsOnDisk = tester.getNumberOfUnits();
-        if (unitsOnDisk == numberOfBCUnits) {
-            System.out.println("Can read right number of units: "+unitsOnDisk);
-            return true;
-        } else {
-            System.out.println("Read wrong number of units: "+unitsOnDisk);
-            return false;
-        }
-    }
+	protected DatabaseLayout db = null;
+	protected int percent = 0;
+	// protected BasenameList bachChannelList;
 
-    
-    class BackChannelUnits{
-          int numberOfUnits;
-          UnitLabel[][] unitLabels;
-          double[] unitTimeSpan; 
-          
-          BackChannelUnits(String unitlabelDir, BasenameList basenameList) throws IOException{
-              this.numberOfUnits = basenameList.getLength();
-              unitLabels = new UnitLabel[this.numberOfUnits][];
-              unitTimeSpan = new double[this.numberOfUnits];
-              for(int i=0; i<this.numberOfUnits; i++){
-                  String fileName =  unitlabelDir+File.separator+basenameList.getName(i)+unitlabelExt;
-                  ESTTrackReader pmFile = new ESTTrackReader( getProp(PMARKDIR)+File.separator
-                          + basenameList.getName(i) + db.getProp(PMEXT));
-                  unitLabels[i]   =  readLabFile(fileName);
-                  unitTimeSpan[i] = pmFile.getTimeSpan();
-              }
-          }
-        
-          int getNumberOfUnits(){
-              return this.numberOfUnits;
-          }
-          
-          UnitLabel[] getUnitLabels(int i){
-              return this.unitLabels[i];
-          }
-          
-          double getTimeSpan(int i){
-              return this.unitTimeSpan[i];
-          }
-    }
-    
-    class UnitLabel{
-        String unitName;
-        double startTime;
-        double endTime;
-        int unitIndex;
-        public UnitLabel(String unitName, double startTime, double endTime, int unitIndex){
-            this.unitName = unitName;
-            this.startTime = startTime;
-            this.endTime = endTime;
-            this.unitIndex = unitIndex;
-        }
-    }
-    
-    /**
-     * @param labFile
-     * @return
-     */
-    private UnitLabel[] readLabFile(String labFile) throws IOException{
-        
-        ArrayList<String> lines = new ArrayList<String>();
-        BufferedReader labels = new BufferedReader
-                            (new InputStreamReader
-                                    (new FileInputStream
-                                            (new File(labFile)), "UTF-8"));
-        String line;
-        
-        // Read Label file first
-        //1. Skip label file header:
-        while ((line = labels.readLine()) != null) {
-            if (line.startsWith("#")) break; // line starting with "#" marks end of header
-        }
-        
-        //2. Put data into an ArrayList  
-        String labelUnit = null;
-        double startTimeStamp = 0.0;
-        double endTimeStamp = 0.0;
-        int unitIndex = 0;
-        while ((line = labels.readLine()) != null) {
-            labelUnit = null;
-            if (line != null){
-                List labelUnitData = getLabelUnitData(line);
-                if(labelUnitData==null) continue;
-                labelUnit = (String)labelUnitData.get(2);
-                unitIndex = Integer.parseInt((String)labelUnitData.get(1));
-                endTimeStamp = Double.parseDouble((String)labelUnitData.get(0)); 
-            }
-            if(labelUnit == null) break;
-            lines.add(labelUnit.trim()+" "+startTimeStamp+" "+endTimeStamp+" "+unitIndex);
-            startTimeStamp = endTimeStamp;  
-        }
-        labels.close();
-        
-        UnitLabel[] ulab = new UnitLabel[lines.size()];
-        Iterator<String> itr = lines.iterator();
-        for(int i=0; itr.hasNext() ; i++) {
-            String element = itr.next();
-            String[] wrds = element.split("\\s+");
-            ulab[i] = new UnitLabel(wrds[0],
-                    (new Double(wrds[1])).doubleValue(),
-                    (new Double(wrds[2])).doubleValue(),
-                    (new Integer(wrds[3])).intValue());
-        }
-        return ulab;
-    }
-    
-    /**
-     * To get Label Unit DATA (time stamp, index, phone unit)
-     * @param line
-     * @return ArrayList contains time stamp, index and phone unit
-     * @throws IOException
-     */
-    private ArrayList getLabelUnitData(String line) throws IOException
-    {
-        if (line == null) return null;
-        if (line.trim().equals("")) return null;
-        ArrayList unitData = new ArrayList();
-        StringTokenizer st = new StringTokenizer(line.trim());
-        //the first token is the time
-        unitData.add(st.nextToken()); 
-        //the second token is the unit index
-        unitData.add(st.nextToken());
-        //the third token is the phone
-        unitData.add(st.nextToken());
-        return unitData;
-    }
-    
-    /**
-     * Provide the progress of computation, in percent, or -1 if
-     * that feature is not implemented.
-     * @return -1 if not implemented, or an integer between 0 and 100.
-     */
-    public int getProgress()
-    {
-        return percent;
-    }
+	public String LABELDIR = "VocalizationUnitfileWriter.backchannelLabDir";
+	public String UNITFILE = "VocalizationUnitfileWriter.unitFile";
+	public final String PMARKDIR = "VocalizationUnitfileWriter.pitchmarkDir";
+	// public String BASELIST = "VocalizationUnitfileWriter.backchannelBaseNamesList";
 
-    public static void main(String[] args) throws Exception
-    {
-        VocalizationUnitfileWriter ufw = new VocalizationUnitfileWriter();
-        new DatabaseLayout(ufw); 
-        ufw.compute();
-    }
+	public final String PMDIR = "db.pmDir";
+	public final String PMEXT = "db.pmExtension";
+
+	public String getName() {
+		return "VocalizationUnitfileWriter";
+	}
+
+	@Override
+	protected void initialiseComp() {
+		// maryDir = new File(db.getProp(db.FILEDIR));
+
+		samplingRate = Integer.parseInt(db.getProp(db.SAMPLINGRATE));
+		// pauseSymbol = System.getProperty("pause.symbol", "pau");
+
+		unitFileName = getProp(UNITFILE);
+		unitlabelDir = new File(getProp(LABELDIR));
+
+		String timelineDir = db.getProp(db.VOCALIZATIONSDIR) + File.separator + "files";
+		if (!(new File(timelineDir)).exists()) {
+
+			System.out.println("vocalizations/files directory does not exist; ");
+			if (!(new File(timelineDir)).mkdirs()) {
+				throw new Error("Could not create vocalizations/files");
+			}
+			System.out.println("Created successfully.\n");
+
+		}
+
+		try {
+			String basenameFile = db.getProp(db.VOCALIZATIONSDIR) + File.separator + "basenames.lst";
+			if ((new File(basenameFile)).exists()) {
+				System.out.println("Loading basenames of vocalizations from '" + basenameFile + "' list...");
+				bnlVocalizations = new BasenameList(basenameFile);
+				System.out.println("Found " + bnlVocalizations.getLength() + " vocalizations in basename list");
+			} else {
+				String vocalWavDir = db.getProp(db.VOCALIZATIONSDIR) + File.separator + "wav";
+				System.out.println("Loading basenames of vocalizations from '" + vocalWavDir + "' directory...");
+				bnlVocalizations = new BasenameList(vocalWavDir, ".wav");
+				System.out.println("Found " + bnlVocalizations.getLength() + " vocalizations in " + vocalWavDir + " directory");
+			}
+		} catch (IOException e) {
+			e.printStackTrace();
+		}
+
+	}
+
+	public SortedMap getDefaultProps(DatabaseLayout db) {
+		this.db = db;
+		if (props == null) {
+			props = new TreeMap();
+			String vocalizationsrootDir = db.getProp(db.VOCALIZATIONSDIR);
+			props.put(LABELDIR, vocalizationsrootDir + File.separator + "lab" + System.getProperty("file.separator"));
+			props.put(
+					UNITFILE,
+					vocalizationsrootDir + File.separator + "files" + File.separator + "vocalization_units"
+							+ db.getProp(db.MARYEXT));
+			props.put(PMARKDIR, db.getProp(db.VOCALIZATIONSDIR) + File.separator + "pm");
+			// props.put(BASELIST, "backchannel.lst");
+		}
+		return props;
+	}
+
+	protected void setupHelp() {
+		props2Help = new TreeMap();
+		props2Help.put(LABELDIR, "directory containing the phone labels");
+		props2Help.put(UNITFILE, "file containing all phone units. Will be created by this module");
+	}
+
+	@Override
+	public boolean compute() throws IOException, MaryConfigurationException {
+		if (!unitlabelDir.exists()) {
+			System.out.print(LABELDIR + " " + getProp(LABELDIR) + " does not exist; ");
+			throw new Error("LABELDIR not found");
+		}
+
+		System.out.println("Back channel unitfile writer started...");
+		BackChannelUnits bcUnits = new BackChannelUnits(unitlabelDir.getAbsolutePath(), this.bnlVocalizations);
+		DataOutputStream out = new DataOutputStream(new BufferedOutputStream(new FileOutputStream(unitFileName)));
+		long posNumUnits = new MaryHeader(MaryHeader.LISTENERUNITS).writeTo(out);
+		int numberOfBCUnits = bcUnits.getNumberOfUnits();
+		out.writeInt(numberOfBCUnits);
+		out.writeInt(samplingRate);
+		long globalStart = 0l; // time, given as sample position with samplingRate
+		for (int i = 0; i < numberOfBCUnits; i++) {
+			UnitLabel[] fileLabels = bcUnits.getUnitLabels(i);
+			double unitTimeSpan = bcUnits.getTimeSpan(i);
+			int localLabLength = fileLabels.length;
+			out.writeInt(localLabLength);
+			for (int j = 0; j < localLabLength; j++) {
+				double startTime = fileLabels[j].startTime;
+				double endTime = fileLabels[j].endTime;
+				double duration = endTime - startTime;
+				long end = (long) (endTime * (double) (samplingRate));
+				long start = (long) (startTime * (double) (samplingRate));
+				out.writeLong(globalStart + start);
+				out.writeInt((int) (end - start));
+				out.writeInt(fileLabels[j].unitName.toCharArray().length);
+				out.writeChars(fileLabels[j].unitName);
+			}
+			globalStart += ((long) ((double) (unitTimeSpan) * (double) (samplingRate)));
+		}
+
+		out.close();
+		VocalizationUnitFileReader tester = new VocalizationUnitFileReader(unitFileName);
+		int unitsOnDisk = tester.getNumberOfUnits();
+		if (unitsOnDisk == numberOfBCUnits) {
+			System.out.println("Can read right number of units: " + unitsOnDisk);
+			return true;
+		} else {
+			System.out.println("Read wrong number of units: " + unitsOnDisk);
+			return false;
+		}
+	}
+
+	class BackChannelUnits {
+		int numberOfUnits;
+		UnitLabel[][] unitLabels;
+		double[] unitTimeSpan;
+
+		BackChannelUnits(String unitlabelDir, BasenameList basenameList) throws IOException {
+			this.numberOfUnits = basenameList.getLength();
+			unitLabels = new UnitLabel[this.numberOfUnits][];
+			unitTimeSpan = new double[this.numberOfUnits];
+			for (int i = 0; i < this.numberOfUnits; i++) {
+				String fileName = unitlabelDir + File.separator + basenameList.getName(i) + unitlabelExt;
+				ESTTrackReader pmFile = new ESTTrackReader(getProp(PMARKDIR) + File.separator + basenameList.getName(i)
+						+ db.getProp(PMEXT));
+				unitLabels[i] = readLabFile(fileName);
+				unitTimeSpan[i] = pmFile.getTimeSpan();
+			}
+		}
+
+		int getNumberOfUnits() {
+			return this.numberOfUnits;
+		}
+
+		UnitLabel[] getUnitLabels(int i) {
+			return this.unitLabels[i];
+		}
+
+		double getTimeSpan(int i) {
+			return this.unitTimeSpan[i];
+		}
+	}
+
+	class UnitLabel {
+		String unitName;
+		double startTime;
+		double endTime;
+		int unitIndex;
+
+		public UnitLabel(String unitName, double startTime, double endTime, int unitIndex) {
+			this.unitName = unitName;
+			this.startTime = startTime;
+			this.endTime = endTime;
+			this.unitIndex = unitIndex;
+		}
+	}
+
+	/**
+	 * @param labFile
+	 * @return
+	 */
+	private UnitLabel[] readLabFile(String labFile) throws IOException {
+
+		ArrayList<String> lines = new ArrayList<String>();
+		BufferedReader labels = new BufferedReader(new InputStreamReader(new FileInputStream(new File(labFile)), "UTF-8"));
+		String line;
+
+		// Read Label file first
+		// 1. Skip label file header:
+		while ((line = labels.readLine()) != null) {
+			if (line.startsWith("#"))
+				break; // line starting with "#" marks end of header
+		}
+
+		// 2. Put data into an ArrayList
+		String labelUnit = null;
+		double startTimeStamp = 0.0;
+		double endTimeStamp = 0.0;
+		int unitIndex = 0;
+		while ((line = labels.readLine()) != null) {
+			labelUnit = null;
+			if (line != null) {
+				List labelUnitData = getLabelUnitData(line);
+				if (labelUnitData == null)
+					continue;
+				labelUnit = (String) labelUnitData.get(2);
+				unitIndex = Integer.parseInt((String) labelUnitData.get(1));
+				endTimeStamp = Double.parseDouble((String) labelUnitData.get(0));
+			}
+			if (labelUnit == null)
+				break;
+			lines.add(labelUnit.trim() + " " + startTimeStamp + " " + endTimeStamp + " " + unitIndex);
+			startTimeStamp = endTimeStamp;
+		}
+		labels.close();
+
+		UnitLabel[] ulab = new UnitLabel[lines.size()];
+		Iterator<String> itr = lines.iterator();
+		for (int i = 0; itr.hasNext(); i++) {
+			String element = itr.next();
+			String[] wrds = element.split("\\s+");
+			ulab[i] = new UnitLabel(wrds[0], (new Double(wrds[1])).doubleValue(), (new Double(wrds[2])).doubleValue(),
+					(new Integer(wrds[3])).intValue());
+		}
+		return ulab;
+	}
+
+	/**
+	 * To get Label Unit DATA (time stamp, index, phone unit)
+	 * 
+	 * @param line
+	 * @return ArrayList contains time stamp, index and phone unit
+	 * @throws IOException
+	 */
+	private ArrayList getLabelUnitData(String line) throws IOException {
+		if (line == null)
+			return null;
+		if (line.trim().equals(""))
+			return null;
+		ArrayList unitData = new ArrayList();
+		StringTokenizer st = new StringTokenizer(line.trim());
+		// the first token is the time
+		unitData.add(st.nextToken());
+		// the second token is the unit index
+		unitData.add(st.nextToken());
+		// the third token is the phone
+		unitData.add(st.nextToken());
+		return unitData;
+	}
+
+	/**
+	 * Provide the progress of computation, in percent, or -1 if that feature is not implemented.
+	 * 
+	 * @return -1 if not implemented, or an integer between 0 and 100.
+	 */
+	public int getProgress() {
+		return percent;
+	}
+
+	public static void main(String[] args) throws Exception {
+		VocalizationUnitfileWriter ufw = new VocalizationUnitfileWriter();
+		new DatabaseLayout(ufw);
+		ufw.compute();
+	}
 
 }
-

marytts-builder/src/main/java/marytts/util/data/text/BasenameClassificationDefinitionFileReader.java

@@ -68,99 +68,99 @@ import org.apache.oro.text.regex.Perl5Matcher;
  * 
  */
 public class BasenameClassificationDefinitionFileReader {
-    protected BufferedReader reader;
+	protected BufferedReader reader;
 
-    public boolean fileOK = true;
+	public boolean fileOK = true;
 
-    protected LinkedHashMap<Pattern, String> styleDefinitions = new LinkedHashMap<Pattern, String>();
+	protected LinkedHashMap<Pattern, String> styleDefinitions = new LinkedHashMap<Pattern, String>();
 
-    /**
-     * constructor to call main constructor with a filename String
-     * 
-     * @param filename
-     *            as a String
-     * @throws IOException
-     */
-    public BasenameClassificationDefinitionFileReader(String filename) throws IOException {
-        this(new FileReader(filename));
-    }
+	/**
+	 * constructor to call main constructor with a filename String
+	 * 
+	 * @param filename
+	 *            as a String
+	 * @throws IOException
+	 */
+	public BasenameClassificationDefinitionFileReader(String filename) throws IOException {
+		this(new FileReader(filename));
+	}
 
-    /**
-     * main constructor
-     * 
-     * @param reader
-     *            as a Reader
-     * @throws IOException
-     */
-    public BasenameClassificationDefinitionFileReader(Reader reader) throws IOException {
-        this.reader = new BufferedReader(reader);
-        parseDefinitionFile();
-    }
+	/**
+	 * main constructor
+	 * 
+	 * @param reader
+	 *            as a Reader
+	 * @throws IOException
+	 */
+	public BasenameClassificationDefinitionFileReader(Reader reader) throws IOException {
+		this.reader = new BufferedReader(reader);
+		parseDefinitionFile();
+	}
 
-    /**
-     * parse style definition file (see class documentation above for format), putting &lt;glob expression, style string&gt; pairs
-     * in styleDefinitions
-     * 
-     * @throws IOException
-     */
-    private void parseDefinitionFile() throws IOException {
-        String line;
-        String globString;
-        String styleString;
-        GlobCompiler glob = new GlobCompiler();
-        Pattern globPattern;
-        // read lines...
-        while ((line = reader.readLine()) != null) {
-            // ...trimming whitespace:
-            line = line.trim();
-            // ignore lines that are empty or start with #:
-            if (line.equals("") || line.startsWith("#")) {
-                continue;
-            } else {
-                // split lines into fields
-                String[] fields = line.split("=");
-                try {
-                    globString = fields[0].trim();
-                    styleString = fields[1].trim();
-                } catch (IndexOutOfBoundsException iob) {
-                    System.err.println("Warning: could not parse line: " + line);
-                    fileOK = false;
-                    continue;
-                }
-                // create GlobCompiler for glob expression:
-                try {
-                    globPattern = glob.compile(globString);
-                } catch (MalformedPatternException mpe) {
-                    System.err.println("Warning: could not parse line: ");
-                    fileOK = false;
-                    continue;
-                }
-                // put (glob expression, style string) pair in styleDefinions:
-                styleDefinitions.put(globPattern, styleString);
-            }
-        }
-        if (styleDefinitions.isEmpty()) {
-            System.err.println("Warning: no style definitions were found!");
-        }
-    }
+	/**
+	 * parse style definition file (see class documentation above for format), putting &lt;glob expression, style string&gt; pairs
+	 * in styleDefinitions
+	 * 
+	 * @throws IOException
+	 */
+	private void parseDefinitionFile() throws IOException {
+		String line;
+		String globString;
+		String styleString;
+		GlobCompiler glob = new GlobCompiler();
+		Pattern globPattern;
+		// read lines...
+		while ((line = reader.readLine()) != null) {
+			// ...trimming whitespace:
+			line = line.trim();
+			// ignore lines that are empty or start with #:
+			if (line.equals("") || line.startsWith("#")) {
+				continue;
+			} else {
+				// split lines into fields
+				String[] fields = line.split("=");
+				try {
+					globString = fields[0].trim();
+					styleString = fields[1].trim();
+				} catch (IndexOutOfBoundsException iob) {
+					System.err.println("Warning: could not parse line: " + line);
+					fileOK = false;
+					continue;
+				}
+				// create GlobCompiler for glob expression:
+				try {
+					globPattern = glob.compile(globString);
+				} catch (MalformedPatternException mpe) {
+					System.err.println("Warning: could not parse line: ");
+					fileOK = false;
+					continue;
+				}
+				// put (glob expression, style string) pair in styleDefinions:
+				styleDefinitions.put(globPattern, styleString);
+			}
+		}
+		if (styleDefinitions.isEmpty()) {
+			System.err.println("Warning: no style definitions were found!");
+		}
+	}
 
-    /**
-     * match basename against the glob expressions in styleDefinitions
-     * 
-     * @param basename
-     * @return style String of first matching glob expression, or empty String if no glob matches
-     */
-    public String getValue(String basename) {
-        Perl5Matcher globMatcher = new Perl5Matcher();
-        String style = "";
-        for (Pattern globPattern : styleDefinitions.keySet()) {
-            if (globMatcher.matches(basename, globPattern)) {
-                style = styleDefinitions.get(globPattern);
-                break; // enable this line to change behavior to return style of *first* matching glob expr
-                // return style; // enable this line to change behavior to return style of *last* matching glob expr
-            }
-        }
-        // no globPattern in styleDefinitions matched... return empty string:
-        return style;
-    }
+	/**
+	 * match basename against the glob expressions in styleDefinitions
+	 * 
+	 * @param basename
+	 * @return style String of first matching glob expression, or empty String if no glob matches
+	 */
+	public String getValue(String basename) {
+		Perl5Matcher globMatcher = new Perl5Matcher();
+		String style = "";
+		for (Pattern globPattern : styleDefinitions.keySet()) {
+			if (globMatcher.matches(basename, globPattern)) {
+				style = styleDefinitions.get(globPattern);
+				break; // enable this line to change behavior to return style of *first* matching glob expr
+				// return style; // enable this line to change behavior to return style of *last* matching glob expr
+			}
+		}
+		// no globPattern in styleDefinitions matched... return empty string:
+		return style;
+	}
 }

marytts-builder/src/main/java/marytts/vocalizations/VocalizationAnnotationReader.java

@@ -32,101 +32,97 @@ import marytts.util.io.BasenameList;
 /**
  * Reader class for manual annotation of vocalizations
  * 
- * The format of the file should be as following examples:  
- * (first line indicates list of feature names)
+ * The format of the file should be as following examples: (first line indicates list of feature names)
+ * 
+ * categories|name|voicequality|angry|sadness|amusement|happiness|contempt|solidarity|antagonism|certain|agreeing|interested|
+ * anticipation Obadiah_049|right|modal|2|3|1|1|2|2|1|0|0|-1|0 Poppy2_078|yeah|modal|1|1|1|2|1|2|1|-1|0|-1|0
+ * Prudence_058|(laughter)|breathy|1|1|3|2|1|2|1|0|0|0|-1 Poppy2_085|yeah|breathy|1|2|1|1|1|2|1|0|0|-1|0
+ * 
  * 
- * categories|name|voicequality|angry|sadness|amusement|happiness|contempt|solidarity|antagonism|certain|agreeing|interested|anticipation
- * Obadiah_049|right|modal|2|3|1|1|2|2|1|0|0|-1|0
- * Poppy2_078|yeah|modal|1|1|1|2|1|2|1|-1|0|-1|0
- * Prudence_058|(laughter)|breathy|1|1|3|2|1|2|1|0|0|0|-1
- * Poppy2_085|yeah|breathy|1|2|1|1|1|2|1|0|0|-1|0
- *  
- *  
  * @author sathish
  *
  */
 public class VocalizationAnnotationReader {
 
-    private ArrayList<String> featureCategories; // feature categories  
-    private Map<String,Map<String, String>> annotationData; // basename --> (feature category, feature value)
-    private BasenameList bnl = null;
-    
-    public VocalizationAnnotationReader(String featureFile) throws IOException {
-        this(featureFile, null);
-    }
-    
-    public VocalizationAnnotationReader(String featureFile, BasenameList bnl) throws IOException {
-        this.bnl = bnl;
-        formatChecker(featureFile);
-        featureCategories = new ArrayList<String>();
-        annotationData    = new HashMap<String,Map<String, String>>();
-        getAnnotations(featureFile); 
-    }
-    
-    /**
-     * 
-     * @param featureFile
-     * @throws IOException 
-     */
-    private void formatChecker(String featureFile) throws IOException {
-        BufferedReader bfrMean = new BufferedReader(new FileReader(new File(featureFile)));
-        String lineMeaning = bfrMean.readLine();
-        String[] mlines = lineMeaning.split("\\|");
-        int noOfStrings = mlines.length;
-        
-        while ( (lineMeaning = bfrMean.readLine()) != null ) {
-            mlines = lineMeaning.split("\\|");
-            if(noOfStrings != mlines.length) {
-                throw new RuntimeException("the format of the file is not good.");
-            }
-        }
-        
-        System.out.println("The format of the file is good");
-        bfrMean.close();
-    }
+	private ArrayList<String> featureCategories; // feature categories
+	private Map<String, Map<String, String>> annotationData; // basename --> (feature category, feature value)
+	private BasenameList bnl = null;
 
-    private void getAnnotations( String featureFile ) throws IOException {
-        
-        BufferedReader bfrMean = new BufferedReader(new FileReader(new File(featureFile)));
-        String lineMeaning = bfrMean.readLine();
-        String[] mlines = lineMeaning.split("\\|");
-        
-        // read feature categories
-        for ( int i=1; i < mlines.length; i++ ) {
-            featureCategories.add(mlines[i].trim());
-        }
-        
-        while ( (lineMeaning = bfrMean.readLine()) != null ) {
-            mlines = lineMeaning.split("\\|");
-            if ( bnl != null ) {
-                if ( !bnl.contains(mlines[0].trim()) ) {
-                    continue;
-                }
-            }
-            Map<String, String> featureMap = new HashMap<String, String>();  
-            // read feature categories
-            for ( int i=1; i < mlines.length; i++ ) {
-                featureMap.put( featureCategories.get(i-1), mlines[i].trim() );
-            }
-            annotationData.put(mlines[0].trim(), featureMap);
-        }
-        bfrMean.close();
-    }
-    
-    public Map<String,Map<String, String>> getVocalizationsAnnotation() {
-        return this.annotationData;
-    }
-    
-    public ArrayList<String> getFeatureList() {
-        return this.featureCategories;
-    }
+	public VocalizationAnnotationReader(String featureFile) throws IOException {
+		this(featureFile, null);
+	}
 
-    /**
-     * @param args
-     */
-    public static void main(String[] args) {
-        
+	public VocalizationAnnotationReader(String featureFile, BasenameList bnl) throws IOException {
+		this.bnl = bnl;
+		formatChecker(featureFile);
+		featureCategories = new ArrayList<String>();
+		annotationData = new HashMap<String, Map<String, String>>();
+		getAnnotations(featureFile);
+	}
 
-    }
+	/**
+	 * 
+	 * @param featureFile
+	 * @throws IOException
+	 */
+	private void formatChecker(String featureFile) throws IOException {
+		BufferedReader bfrMean = new BufferedReader(new FileReader(new File(featureFile)));
+		String lineMeaning = bfrMean.readLine();
+		String[] mlines = lineMeaning.split("\\|");
+		int noOfStrings = mlines.length;
+
+		while ((lineMeaning = bfrMean.readLine()) != null) {
+			mlines = lineMeaning.split("\\|");
+			if (noOfStrings != mlines.length) {
+				throw new RuntimeException("the format of the file is not good.");
+			}
+		}
+
+		System.out.println("The format of the file is good");
+		bfrMean.close();
+	}
+
+	private void getAnnotations(String featureFile) throws IOException {
+
+		BufferedReader bfrMean = new BufferedReader(new FileReader(new File(featureFile)));
+		String lineMeaning = bfrMean.readLine();
+		String[] mlines = lineMeaning.split("\\|");
+
+		// read feature categories
+		for (int i = 1; i < mlines.length; i++) {
+			featureCategories.add(mlines[i].trim());
+		}
+
+		while ((lineMeaning = bfrMean.readLine()) != null) {
+			mlines = lineMeaning.split("\\|");
+			if (bnl != null) {
+				if (!bnl.contains(mlines[0].trim())) {
+					continue;
+				}
+			}
+			Map<String, String> featureMap = new HashMap<String, String>();
+			// read feature categories
+			for (int i = 1; i < mlines.length; i++) {
+				featureMap.put(featureCategories.get(i - 1), mlines[i].trim());
+			}
+			annotationData.put(mlines[0].trim(), featureMap);
+		}
+		bfrMean.close();
+	}
+
+	public Map<String, Map<String, String>> getVocalizationsAnnotation() {
+		return this.annotationData;
+	}
+
+	public ArrayList<String> getFeatureList() {
+		return this.featureCategories;
+	}
+
+	/**
+	 * @param args
+	 */
+	public static void main(String[] args) {
+
+	}
 
 }

marytts-builder/src/test/java/marytts/tools/analysis/CopySynthesisTest.java

@@ -21,30 +21,30 @@ import org.w3c.dom.Document;
  */
 public class CopySynthesisTest {
 
-	private AllophoneSet getAllophoneSet() throws MaryConfigurationException  {
-		return AllophoneSet.getAllophoneSet(getClass().getResourceAsStream("/marytts/language/en_GB/lexicon/allophones.en_GB.xml"), "dummy");
+	private AllophoneSet getAllophoneSet() throws MaryConfigurationException {
+		return AllophoneSet.getAllophoneSet(getClass()
+				.getResourceAsStream("/marytts/language/en_GB/lexicon/allophones.en_GB.xml"), "dummy");
 	}
-	
+
 	private Labels getReferenceLabels() throws IOException {
 		return new Labels(getClass().getResourceAsStream("pop001.lab"));
 	}
-	
+
 	private Document getTestDocument() throws Exception {
 		return DomUtils.parseDocument(getClass().getResourceAsStream("pop001.dfki-poppy.ACOUSTPARAMS"));
 	}
-	
+
 	private void assertSimilarDurations(Labels l1, Labels l2) {
 		// label durations are "similar" if for each label pair the difference is less than one millisecond
 		assertEquals(l1.items.length, l2.items.length);
 		assertEquals(l1.items[0].time, l2.items[0].time, 0.001);
-		for (int i=1; i<l1.items.length; i++) {
-			double d1 = l1.items[i].time - l1.items[i-1].time;
-			double d2 = l2.items[i].time - l2.items[i-1].time;
+		for (int i = 1; i < l1.items.length; i++) {
+			double d1 = l1.items[i].time - l1.items[i - 1].time;
+			double d2 = l2.items[i].time - l2.items[i - 1].time;
 			assertEquals(d1, d2, 0.001);
 		}
 	}
-	
-	
+
 	@Test
 	public void imposeSegments() throws Exception {
 		Labels source = getReferenceLabels();
@@ -53,7 +53,7 @@ public class CopySynthesisTest {
 		cs.imposeSegments(source, target);
 		assertArrayEquals(source.getLabelSymbols(), new Labels(target).getLabelSymbols());
 	}
-	
+
 	@Test
 	public void imposeDurations() throws Exception {
 		Labels source = getReferenceLabels();

marytts-builder/src/test/java/marytts/tools/dbselection/CoverageDefinitionIT.java

@@ -33,19 +33,14 @@ import org.junit.Test;
  */
 public class CoverageDefinitionIT {
 
-	private static String[] testSentences = new String[] {
-		"Author of the danger trail, Philip Steels, etc.",
-		"Not at this particular case, Tom, apologized Whittemore.",
-		"For the twentieth time that evening the two men shook hands.",
-		"Lord, but I'm glad to see you again, Phil.",
-		"Will we ever forget it.",
-		"God bless 'em, I hope I'll go on seeing them forever.",
-		"And you always want to see it in the superlative degree.",
-		"Gad, your letter came just in time.",
-		"He turned sharply, and faced Gregson across the table.",
-		"I'm playing a single hand in what looks like a losing game."
-	};
-	
+	private static String[] testSentences = new String[] { "Author of the danger trail, Philip Steels, etc.",
+			"Not at this particular case, Tom, apologized Whittemore.",
+			"For the twentieth time that evening the two men shook hands.", "Lord, but I'm glad to see you again, Phil.",
+			"Will we ever forget it.", "God bless 'em, I hope I'll go on seeing them forever.",
+			"And you always want to see it in the superlative degree.", "Gad, your letter came just in time.",
+			"He turned sharply, and faced Gregson across the table.",
+			"I'm playing a single hand in what looks like a losing game." };
+
 	private static byte[][] coverageFeatures = new byte[testSentences.length][];
 	private static Locale locale = Locale.US;
 	private static String featureNames = "phone next_phone selection_prosody";
@@ -53,12 +48,12 @@ public class CoverageDefinitionIT {
 
 	@BeforeClass
 	public static void setup() throws Exception {
-		for (int i=0; i<testSentences.length; i++) {
+		for (int i = 0; i < testSentences.length; i++) {
 			coverageFeatures[i] = CoverageUtils.sentenceToFeatures(testSentences[i], locale, featureNames, false);
 		}
 		featDef = FeatureRegistry.getTargetFeatureComputer(locale, featureNames).getFeatureDefinition();
 	}
-	
+
 	@Test
 	public void testAll() throws Exception {
 		CoverageFeatureProvider cfProvider = new InMemoryCFProvider(coverageFeatures, null);

marytts-builder/src/test/java/marytts/tools/dbselection/CoverageDefinitionTest.java

@@ -37,10 +37,11 @@ import static org.junit.Assert.*;
 public class CoverageDefinitionTest {
 	private FeatureDefinition featDef;
 	private CoverageFeatureProvider cfProvider;
-	
+
 	@Before
 	public void setup() throws Exception {
-		String targetFeaturesData = FileUtils.getStreamAsString(getClass().getResourceAsStream("helloworld.targetfeatures"), "UTF-8");
+		String targetFeaturesData = FileUtils.getStreamAsString(getClass().getResourceAsStream("helloworld.targetfeatures"),
+				"UTF-8");
 		featDef = FeatureUtils.readFeatureDefinition(targetFeaturesData);
 		FeatureVector[] features = FeatureUtils.readFeatureVectors(targetFeaturesData);
 		byte[][] data = new byte[1][];

marytts-builder/src/test/java/marytts/tools/dbselection/CoverageUtilsIT.java

@@ -39,10 +39,10 @@ public class CoverageUtilsIT {
 
 	String text = "Hello world";
 	Locale locale = Locale.US;
-	String[] phones = {"h", "@", "l", "@U", "w", "r=", "l", "d", "_"};
-	String[] next_phones = {"@", "l", "@U", "w", "r=", "l", "d", "_", "_"};
-	String[] selection_prosodies = {"0", "0", "pre-nuclear", "pre-nuclear", "nuclear", "nuclear", "nuclear", "nuclear", "0" };
-	
+	String[] phones = { "h", "@", "l", "@U", "w", "r=", "l", "d", "_" };
+	String[] next_phones = { "@", "l", "@U", "w", "r=", "l", "d", "_", "_" };
+	String[] selection_prosodies = { "0", "0", "pre-nuclear", "pre-nuclear", "nuclear", "nuclear", "nuclear", "nuclear", "0" };
+
 	private void assertFeaturesMatchTarget(byte[] data, String featureNames) {
 		assertNotNull(data);
 		String[] featureNameArray = featureNames.split(" ");
@@ -51,7 +51,7 @@ public class CoverageUtilsIT {
 		assertEquals(0, data.length % numFeatures);
 		TargetFeatureComputer tfc = FeatureRegistry.getTargetFeatureComputer(locale, featureNames);
 		FeatureDefinition def = tfc.getFeatureDefinition();
-		for (int k=0; k<numFeatures; k++) {
+		for (int k = 0; k < numFeatures; k++) {
 			String featureName = featureNameArray[k];
 			String[] possibleValues = def.getPossibleValues(def.getFeatureIndex(featureName));
 			String[] values;
@@ -62,25 +62,24 @@ public class CoverageUtilsIT {
 			} else if (featureName.equals("selection_prosody")) {
 				values = selection_prosodies;
 			} else {
-				throw new IllegalArgumentException("Unexpected feature name: "+featureName);
+				throw new IllegalArgumentException("Unexpected feature name: " + featureName);
 			}
-			for (int i=0; i<phones.length; i++) {
-				int pos = i*numFeatures + k;
+			for (int i = 0; i < phones.length; i++) {
+				int pos = i * numFeatures + k;
 				String actual = possibleValues[data[pos]];
 				String expected = values[i];
 				assertEquals(expected, actual);
 			}
 		}
 	}
-	
-	
+
 	@BeforeClass
 	public static void startMary() throws Exception {
 		if (Mary.currentState() == Mary.STATE_OFF) {
 			Mary.startup();
 		}
 	}
-	
+
 	@Test
 	public void canComputePhoneFeature() throws Exception {
 		// setup
@@ -91,7 +90,7 @@ public class CoverageUtilsIT {
 		assertFeaturesMatchTarget(data, featureNames);
 	}
 
-	@Test(expected=SynthesisException.class)
+	@Test(expected = SynthesisException.class)
 	public void willRejectUnknownFeature() throws Exception {
 		// setup
 		String featureNames = "unknown";

marytts-builder/src/test/java/marytts/tools/newlanguage/LTSTrainerTest.java

@@ -36,7 +36,7 @@ public class LTSTrainerTest {
 		InputStream alloStream = LTSTrainerTest.class.getResourceAsStream("allophones.it.xml");
 		AllophoneSet allophoneset;
 		try {
-			
+
 			allophoneset = AllophoneSet.getAllophoneSet(alloStream, "test");
 
 			// Initialize trainer
@@ -54,16 +54,16 @@ public class LTSTrainerTest {
 				tp.alignIteration();
 			}
 			CART st = tp.trainTree(10);
-			
-			// Temp file for test 
+
+			// Temp file for test
 			String OutputFilename = "LTS_test.tree";
 			File tempFile = testFolder.newFile(OutputFilename);
-			System.out.println("Writing tree in tempFile: " + tempFile.getAbsolutePath()+tempFile.getName());
+			System.out.println("Writing tree in tempFile: " + tempFile.getAbsolutePath() + tempFile.getName());
+
+			tp.save(st, tempFile.getAbsolutePath() + tempFile.getName());
 
-			tp.save(st, tempFile.getAbsolutePath()+tempFile.getName());
-			
 			assertTrue(tempFile.exists());
-			
+
 		} catch (IOException e) {
 			// TODO Auto-generated catch block
 			e.printStackTrace();

marytts-builder/src/test/java/marytts/tools/voiceimport/DatabaseLayoutTest.java

@@ -19,11 +19,9 @@ public class DatabaseLayoutTest {
 	File dummyConfigFile;
 	DatabaseLayout db;
 
-	private static File createDummyConfigFile(TemporaryFolder parent)
-			throws IOException {
-		BufferedReader br = new BufferedReader(
-				new InputStreamReader(DatabaseLayoutTest.class
-						.getResourceAsStream("database.config"), "UTF-8"));
+	private static File createDummyConfigFile(TemporaryFolder parent) throws IOException {
+		BufferedReader br = new BufferedReader(new InputStreamReader(
+				DatabaseLayoutTest.class.getResourceAsStream("database.config"), "UTF-8"));
 		File conf = parent.newFile("dummy.config");
 		PrintWriter out = new PrintWriter(conf, "UTF-8");
 		String line;
@@ -47,9 +45,8 @@ public class DatabaseLayoutTest {
 		dummyConfigFile = createDummyConfigFile(dummyVoiceDir);
 		assertTrue(dummyConfigFile.length() > 0);
 		dummyVoiceDir.newFolder("wav");
-		db = new DatabaseLayout(dummyConfigFile, new VoiceImportComponent[] {
-				new AllophonesExtractor(), new BasenameTimelineMaker(),
-				new EHMMLabeler() });
+		db = new DatabaseLayout(dummyConfigFile, new VoiceImportComponent[] { new AllophonesExtractor(),
+				new BasenameTimelineMaker(), new EHMMLabeler() });
 	}
 
 	@Test

marytts-builder/src/test/java/marytts/unitselection/data/TestableTimelineReader.java

@@ -30,25 +30,24 @@ import marytts.util.data.Datagram;
  * Provides the actual timeline test case for the timeline reading/writing symmetry.
  */
 public class TestableTimelineReader extends TimelineReader {
-    
-    public TestableTimelineReader(String fileName, boolean tryMemoryMapping) throws Exception {
-        super(fileName, tryMemoryMapping);
-    }
 
-    @Override
-    public Pair<ByteBuffer, Long> getByteBufferAtTime(long targetTimeInSamples) throws IOException, BufferUnderflowException {
-        return super.getByteBufferAtTime(targetTimeInSamples);
-    }
+	public TestableTimelineReader(String fileName, boolean tryMemoryMapping) throws Exception {
+		super(fileName, tryMemoryMapping);
+	}
 
-    @Override
-    public long skipNextDatagram(ByteBuffer bb) throws IOException {
-        return super.skipNextDatagram(bb);
-    }
+	@Override
+	public Pair<ByteBuffer, Long> getByteBufferAtTime(long targetTimeInSamples) throws IOException, BufferUnderflowException {
+		return super.getByteBufferAtTime(targetTimeInSamples);
+	}
 
-    @Override
-    public Datagram getNextDatagram(ByteBuffer bb) {
-        return super.getNextDatagram(bb);
-    }
+	@Override
+	public long skipNextDatagram(ByteBuffer bb) throws IOException {
+		return super.skipNextDatagram(bb);
+	}
+
+	@Override
+	public Datagram getNextDatagram(ByteBuffer bb) {
+		return super.getNextDatagram(bb);
+	}
 
 }
-

marytts-builder/src/test/java/marytts/unitselection/data/TimelineTest.java

@@ -41,488 +41,477 @@ import org.junit.Test;
  * Provides the actual timeline test case for the timeline reading/writing symmetry.
  */
 public class TimelineTest {
-    private static TestableTimelineReader tlr;
-    private static String hdrContents;
-    private static int NUMDATAGRAMS;
-    private static int MAXDATAGRAMBYTESIZE;
-    private static int MAXDATAGRAMDURATION;
-    private static int sampleRate;
-    private static Datagram[] origDatagrams;
-    private static final String tlFileName = "timelineTest.bin";
-    
-    
-    @BeforeClass
-    public static void setUp() throws Exception {
-        Random rand = new Random(); // New random number generator
-        
-        NUMDATAGRAMS = rand.nextInt( 87 ) + 6; // Number of datagrams to test with, between 6 and 100.
-        System.out.println( "Testing with [" + NUMDATAGRAMS + "] random datagrams." );
-        MAXDATAGRAMBYTESIZE = 64; // Maximum datagram length in bytes
-        MAXDATAGRAMDURATION = 20; // Maximum datagram duration (in samples)
-        hdrContents = "Blah This is the procHeader Blah";
-        sampleRate = 1000;
-        
-        origDatagrams = new Datagram[NUMDATAGRAMS]; // An array of datagrams with random length
-        int len = 0;
-        long dur = 0l;
-        
-        
-        /* Fill the array of random datagrams */
-        long lenCumul = 74;
-        long durCumul = 0l;
-        for ( int i = 0; i < NUMDATAGRAMS; i++ ) {
-            /* Make the first datagram very long, for special tests */
-            if (i==0 || i==2) {
-                len = 1234567;
-            } else {
-                /* Make a random length */
-                len = rand.nextInt(MAXDATAGRAMBYTESIZE) + 1;
-            }
-            lenCumul += (len + 12);
-            /* Allocate the corresponding byte array */
-            byte[] buff = new byte[len];
-            /* Fill all the bytes with the datagram index */
-            for ( int l = 0; l < len; l++ ) {
-                buff[l] = (byte) i;
-            }
-            /* Make a random datagram duration */
-            dur = (long)( rand.nextInt(MAXDATAGRAMDURATION) + 2 );
-            durCumul += dur;
-            /* Store the datagram */
-            origDatagrams[i] = new Datagram( dur, buff );
-            /* Check */
-            System.out.println( "[ " + len + " , " + dur + " ]\t( " + lenCumul + " , " + durCumul + " )" );
-        }
-        
-        /* Write the datagram array in a timeline */
-        System.out.println( "Opening new timeline file..." );
-        TimelineWriter tlw = new TimelineWriter( tlFileName , hdrContents, sampleRate, 0.1d );
-        System.out.println( "Feeding..." );
-        tlw.feed( origDatagrams, sampleRate );
-        System.out.println( "Closing..." );
-        tlw.close();
-        System.out.println( "Done." );
-        
-        System.out.println( "Datagram zone pos. = " + tlw.getDatagramsBytePos() );
-        System.out.println( "WRITTEN INDEX:" );
-        tlw.getIndex().print();
-        /* Testing the readonly file opening */
-        System.out.println( "Testing the TimelineReader construction..." );
-        /* Re-read the datagrams */
-        tlr = new TestableTimelineReader(tlFileName, true);
-    }
-    
-    
-    @Test
-    public void procHeader() throws IOException {
-        /* Check the procHeader */
-        Assert.assertEquals( "The procHeader is out of sync.", tlr.getProcHeaderContents(), hdrContents );
-    }
+	private static TestableTimelineReader tlr;
+	private static String hdrContents;
+	private static int NUMDATAGRAMS;
+	private static int MAXDATAGRAMBYTESIZE;
+	private static int MAXDATAGRAMDURATION;
+	private static int sampleRate;
+	private static Datagram[] origDatagrams;
+	private static final String tlFileName = "timelineTest.bin";
 
-    @Test
-    public void numDatagrams() throws IOException {
-        /* Check the number of datagrams */
-        Assert.assertEquals( "numDatagrams is out of sync.", tlr.getNumDatagrams(), NUMDATAGRAMS );
-    }
+	@BeforeClass
+	public static void setUp() throws Exception {
+		Random rand = new Random(); // New random number generator
 
+		NUMDATAGRAMS = rand.nextInt(87) + 6; // Number of datagrams to test with, between 6 and 100.
+		System.out.println("Testing with [" + NUMDATAGRAMS + "] random datagrams.");
+		MAXDATAGRAMBYTESIZE = 64; // Maximum datagram length in bytes
+		MAXDATAGRAMDURATION = 20; // Maximum datagram duration (in samples)
+		hdrContents = "Blah This is the procHeader Blah";
+		sampleRate = 1000;
 
-    @Test
-    public void testSkip() throws IOException {
-        System.out.println( "READ INDEX:" );
-        tlr.getIndex().print();
-        /* Testing skip */
-        System.out.println( "Testing skip..." );
-        long timeNow = 0;
-        long timeBefore = 0;
-        Pair<ByteBuffer, Long> p = tlr.getByteBufferAtTime(timeNow);
-        ByteBuffer bb = p.getFirst();
-        int byteNow = bb.position();
-        int byteBefore = 0;
-        for ( int i = 0; i < NUMDATAGRAMS; i++ ) {
-            timeBefore = timeNow;
-            byteBefore = byteNow;
-            try {
-                long skippedDuration = tlr.skipNextDatagram(bb);
-                timeNow += skippedDuration;
-            } catch (BufferUnderflowException e) {
-                // reached end of byte buffer
-                break;
-            }
-            byteNow = bb.position();
-            Assert.assertEquals( "Skipping fails on datagram [" + i + "].", (long)(origDatagrams[i].getLength()) + 12l, (byteNow - byteBefore) );
-            Assert.assertEquals( "Time is out of sync after skipping datagram [" + i + "].", origDatagrams[i].getDuration(), (timeNow - timeBefore) );
-        }
-        
-        /* Testing the EOF trap for skip */
-        try {
-            tlr.skipNextDatagram(bb);
-            Assert.fail("should have thrown BufferUnderflowException to indicate end of byte buffer");
-        } catch(BufferUnderflowException e) {
-            // OK, expected
-        }
+		origDatagrams = new Datagram[NUMDATAGRAMS]; // An array of datagrams with random length
+		int len = 0;
+		long dur = 0l;
 
-    }
+		/* Fill the array of random datagrams */
+		long lenCumul = 74;
+		long durCumul = 0l;
+		for (int i = 0; i < NUMDATAGRAMS; i++) {
+			/* Make the first datagram very long, for special tests */
+			if (i == 0 || i == 2) {
+				len = 1234567;
+			} else {
+				/* Make a random length */
+				len = rand.nextInt(MAXDATAGRAMBYTESIZE) + 1;
+			}
+			lenCumul += (len + 12);
+			/* Allocate the corresponding byte array */
+			byte[] buff = new byte[len];
+			/* Fill all the bytes with the datagram index */
+			for (int l = 0; l < len; l++) {
+				buff[l] = (byte) i;
+			}
+			/* Make a random datagram duration */
+			dur = (long) (rand.nextInt(MAXDATAGRAMDURATION) + 2);
+			durCumul += dur;
+			/* Store the datagram */
+			origDatagrams[i] = new Datagram(dur, buff);
+			/* Check */
+			System.out.println("[ " + len + " , " + dur + " ]\t( " + lenCumul + " , " + durCumul + " )");
+		}
 
+		/* Write the datagram array in a timeline */
+		System.out.println("Opening new timeline file...");
+		TimelineWriter tlw = new TimelineWriter(tlFileName, hdrContents, sampleRate, 0.1d);
+		System.out.println("Feeding...");
+		tlw.feed(origDatagrams, sampleRate);
+		System.out.println("Closing...");
+		tlw.close();
+		System.out.println("Done.");
 
+		System.out.println("Datagram zone pos. = " + tlw.getDatagramsBytePos());
+		System.out.println("WRITTEN INDEX:");
+		tlw.getIndex().print();
+		/* Testing the readonly file opening */
+		System.out.println("Testing the TimelineReader construction...");
+		/* Re-read the datagrams */
+		tlr = new TestableTimelineReader(tlFileName, true);
+	}
 
-    @Test
-    public void testGet() throws IOException {
+	@Test
+	public void procHeader() throws IOException {
+		/* Check the procHeader */
+		Assert.assertEquals("The procHeader is out of sync.", tlr.getProcHeaderContents(), hdrContents);
+	}
 
-        /* Testing get */
-        System.out.println( "Testing get..." );
-        Datagram[] readDatagrams = new Datagram[NUMDATAGRAMS];
-        Pair<ByteBuffer, Long> p = tlr.getByteBufferAtTime(0);
-        ByteBuffer bb = p.getFirst();
-        for ( int i = 0; i < NUMDATAGRAMS; i++ ) {
-            readDatagrams[i] = tlr.getNextDatagram(bb);
-            if (readDatagrams[i] == null) {
-                System.err.println("Could read "+i+" datagrams");
-                break;
-            }
-            Assert.assertTrue( "Datagram [" + i + "] is out of sync.", areEqual( origDatagrams[i].getData(), readDatagrams[i].getData() ) );
-            Assert.assertEquals( "Time for datagram [" + i + "] is out of sync.", origDatagrams[i].getDuration(), readDatagrams[i].getDuration() );
-        }
-        /* Testing the EOF trap for get */
-        Assert.assertEquals( null, tlr.getNextDatagram(bb) );
-    }
-    
-    @Test
-    public void gotoTime1() throws IOException {
-        // setup
-        final int testIdx = NUMDATAGRAMS / 2;
-        long onTime = getTimeOfIndex(testIdx);
-        // exercise
-        ByteBuffer bb = tlr.getByteBufferAtTime(onTime).getFirst();
-        Datagram d = tlr.getNextDatagram(bb);
-        // verify
-        assertEquals(origDatagrams[testIdx], d);
-    }
-    
-    @Test
-    public void gotoTime2() throws IOException {
-        // setup
-        final int testIdx = NUMDATAGRAMS / 2;
-        long onTime = getTimeOfIndex(testIdx);
-        long afterTime = onTime + origDatagrams[testIdx].getDuration();
-        long midTime = onTime + ((afterTime - onTime) / 2);
-        // exercise
-        ByteBuffer bb = tlr.getByteBufferAtTime(midTime).getFirst();
-        Datagram d = tlr.getNextDatagram(bb);
-        // verify
-        assertEquals(origDatagrams[testIdx], d);
-    }
-    
-    @Test
-    public void gotoTime3() throws IOException {
-        // setup
-        final int testIdx = NUMDATAGRAMS / 2;
-        long onTime = getTimeOfIndex(testIdx);
-        long afterTime = onTime + origDatagrams[testIdx].getDuration();
-        // exercise
-        ByteBuffer bb = tlr.getByteBufferAtTime(afterTime).getFirst();
-        Datagram d = tlr.getNextDatagram(bb);
-        // verify
-        assertEquals(origDatagrams[testIdx+1], d);
-    }
+	@Test
+	public void numDatagrams() throws IOException {
+		/* Check the number of datagrams */
+		Assert.assertEquals("numDatagrams is out of sync.", tlr.getNumDatagrams(), NUMDATAGRAMS);
+	}
 
-    @Test
-    public void getDatagrams1() throws IOException {
-        // setup
-        final int testIdx = NUMDATAGRAMS / 2;
-        long onTime = getTimeOfIndex(testIdx);
-        Datagram[] D = null;
-        long span = origDatagrams[testIdx].getDuration();
-        long[] offset = new long[1];
-        // exercise
-        D = tlr.getDatagrams( onTime, span, sampleRate, offset );
-        // verify
-        assertEquals( 1, D.length );
-        assertEquals( origDatagrams[testIdx], D[0] );
-        assertEquals( 0l, offset[0] );
-    }
-    
+	@Test
+	public void testSkip() throws IOException {
+		System.out.println("READ INDEX:");
+		tlr.getIndex().print();
+		/* Testing skip */
+		System.out.println("Testing skip...");
+		long timeNow = 0;
+		long timeBefore = 0;
+		Pair<ByteBuffer, Long> p = tlr.getByteBufferAtTime(timeNow);
+		ByteBuffer bb = p.getFirst();
+		int byteNow = bb.position();
+		int byteBefore = 0;
+		for (int i = 0; i < NUMDATAGRAMS; i++) {
+			timeBefore = timeNow;
+			byteBefore = byteNow;
+			try {
+				long skippedDuration = tlr.skipNextDatagram(bb);
+				timeNow += skippedDuration;
+			} catch (BufferUnderflowException e) {
+				// reached end of byte buffer
+				break;
+			}
+			byteNow = bb.position();
+			Assert.assertEquals("Skipping fails on datagram [" + i + "].", (long) (origDatagrams[i].getLength()) + 12l,
+					(byteNow - byteBefore));
+			Assert.assertEquals("Time is out of sync after skipping datagram [" + i + "].", origDatagrams[i].getDuration(),
+					(timeNow - timeBefore));
+		}
 
-    @Test
-    public void getDatagrams2() throws IOException {
-        // setup
-        final int testIdx = NUMDATAGRAMS / 2;
-        long onTime = getTimeOfIndex(testIdx);
-        Datagram[] D = null;
-        long span = origDatagrams[testIdx].getDuration() / 2;
-        long[] offset = new long[1];
-        // exercise
-        D = tlr.getDatagrams( onTime, span, sampleRate, offset );
-        // verify
-        assertEquals( 1, D.length );
-        assertEquals( origDatagrams[testIdx], D[0] );
-        assertEquals( 0l, offset[0] );
-    }
+		/* Testing the EOF trap for skip */
+		try {
+			tlr.skipNextDatagram(bb);
+			Assert.fail("should have thrown BufferUnderflowException to indicate end of byte buffer");
+		} catch (BufferUnderflowException e) {
+			// OK, expected
+		}
 
-    @Test
-    public void getDatagrams3() throws IOException {
-        // setup
-        final int testIdx = NUMDATAGRAMS / 2;
-        long onTime = getTimeOfIndex(testIdx);
-        long afterTime = onTime + origDatagrams[testIdx].getDuration();
-        long midTime = onTime + ((afterTime - onTime) / 2);
-        Datagram[] D = null;
-        long span = origDatagrams[testIdx].getDuration() / 2;
-        // exercise
-        D = tlr.getDatagrams( midTime, span, sampleRate);
-        // verify
-        assertEquals( 1, D.length );
-        assertEquals( origDatagrams[testIdx], D[0] );
-    }
+	}
 
-    @Test
-    public void getDatagrams4() throws IOException {
-        // setup
-        final int testIdx = NUMDATAGRAMS / 2;
-        long onTime = getTimeOfIndex(testIdx);
-        Datagram[] D = null;
-        long span = origDatagrams[testIdx].getDuration() + 1;
-        long[] offset = new long[1];
-        // exercise
-        D = tlr.getDatagrams( onTime, span, sampleRate, offset );
-        // verify
-        assertEquals( 2, D.length );
-        assertEquals( origDatagrams[testIdx], D[0] );
-        assertEquals( origDatagrams[testIdx+1], D[1] );
-        assertEquals( 0l, offset[0] );
-    }
-    
-    @Test
-    public void getDatagrams5() throws IOException {
-        // setup
-        final int testIdx = NUMDATAGRAMS / 2;
-        long onTime = getTimeOfIndex(testIdx);
-        Datagram[] D = null;
-        long span = origDatagrams[testIdx].getDuration() + origDatagrams[testIdx+1].getDuration();
-        long[] offset = new long[1];
-        // exercise
-        D = tlr.getDatagrams( onTime, span, sampleRate, offset );
-        // verify
-        assertEquals( 2, D.length );
-        assertEquals( origDatagrams[testIdx], D[0] );
-        assertEquals( origDatagrams[testIdx+1], D[1] );
-        assertEquals( 0l, offset[0] );
-    }
-    
-    @Test
-    public void getDatagrams6() throws IOException {
-        // setup
-        final int testIdx = NUMDATAGRAMS / 2;
-        long onTime = getTimeOfIndex(testIdx);
-        Datagram[] D = null;
-        long span = origDatagrams[testIdx].getDuration() + origDatagrams[testIdx+1].getDuration();
-        long[] offset = new long[1];
-        // exercise
-        D = tlr.getDatagrams( onTime+1, span, sampleRate, offset );
-        // verify
-        assertEquals("textIdx="+testIdx+", span="+span+", dur["+testIdx+"]="+origDatagrams[testIdx].getDuration()
-                +", dur["+(testIdx+1)+"]="+origDatagrams[testIdx+1].getDuration()+", offset="+offset[0],
-                3, D.length );
-        assertEquals( origDatagrams[testIdx], D[0] );
-        assertEquals( origDatagrams[testIdx+1], D[1] );
-        assertEquals( origDatagrams[testIdx+2], D[2] );
-        assertEquals( 1l, offset[0] );
-    }
-    
-    @Test
-    public void getDatagrams7() throws IOException {
-        // setup
-        final int testIdx = NUMDATAGRAMS / 2;
-        long onTime = getTimeOfIndex(testIdx);
-        long afterTime = onTime + origDatagrams[testIdx].getDuration();
-        long midTime = onTime + ((afterTime - onTime) / 2);
-        Datagram[] D = null;
-        long dur = origDatagrams[testIdx].getDuration();
-        long span = dur - dur/2 + 1;
-        long[] offset = new long[1];
-        // exercise
-        D = tlr.getDatagrams( midTime, span, sampleRate, offset );
-        // verify
-        assertEquals( 2, D.length );
-        assertEquals( origDatagrams[testIdx], D[0] );
-        assertEquals( origDatagrams[testIdx+1], D[1] );
-        assertEquals( dur/2, offset[0] );
-    }
-    
-    
-    @Test
-    public void otherSampleRate() throws IOException {
-        // setup
-        final int testIdx = NUMDATAGRAMS / 2;
-        long onTime = getTimeOfIndex(testIdx);
-        Datagram[] D = null;
-        long dur = origDatagrams[testIdx].getDuration();
-        long span = dur;
-        // exercise
-        D = tlr.getDatagrams( onTime*2, span*2, sampleRate/2 );
-        // verify
-        assertEquals( 1, D.length );
-        Assert.assertTrue( areEqual(D[0].getData(), origDatagrams[testIdx].getData()));
-        Assert.assertTrue( D[0].getDuration() != origDatagrams[testIdx].getDuration());
-        
-    }
+	@Test
+	public void testGet() throws IOException {
 
-    /**
-     * @param testIdx
-     * @return
-     */
-    private long getTimeOfIndex(final int testIdx) {
-        long onTime = 0l;
-        for( int i = 0; i < testIdx; i++ ) {
-            onTime += origDatagrams[i].getDuration();
-        }
-        return onTime;
-    }
+		/* Testing get */
+		System.out.println("Testing get...");
+		Datagram[] readDatagrams = new Datagram[NUMDATAGRAMS];
+		Pair<ByteBuffer, Long> p = tlr.getByteBufferAtTime(0);
+		ByteBuffer bb = p.getFirst();
+		for (int i = 0; i < NUMDATAGRAMS; i++) {
+			readDatagrams[i] = tlr.getNextDatagram(bb);
+			if (readDatagrams[i] == null) {
+				System.err.println("Could read " + i + " datagrams");
+				break;
+			}
+			Assert.assertTrue("Datagram [" + i + "] is out of sync.",
+					areEqual(origDatagrams[i].getData(), readDatagrams[i].getData()));
+			Assert.assertEquals("Time for datagram [" + i + "] is out of sync.", origDatagrams[i].getDuration(),
+					readDatagrams[i].getDuration());
+		}
+		/* Testing the EOF trap for get */
+		Assert.assertEquals(null, tlr.getNextDatagram(bb));
+	}
 
+	@Test
+	public void gotoTime1() throws IOException {
+		// setup
+		final int testIdx = NUMDATAGRAMS / 2;
+		long onTime = getTimeOfIndex(testIdx);
+		// exercise
+		ByteBuffer bb = tlr.getByteBufferAtTime(onTime).getFirst();
+		Datagram d = tlr.getNextDatagram(bb);
+		// verify
+		assertEquals(origDatagrams[testIdx], d);
+	}
 
-    @Test
-    public void getLastDatagram() throws MaryConfigurationException, IOException {
-        long totalDur = tlr.getTotalDuration();
-        Assert.assertTrue(totalDur > 0);
-        
-        Datagram d = tlr.getDatagram(totalDur - 1);
-        Assert.assertTrue(d != null);
-        
-        long dur = d.getDuration();
-        
-        d = tlr.getDatagram(totalDur - dur);
-        Assert.assertTrue(d != null);
-    }
-    
-    @Test
-    public void getLastDatagrams() throws MaryConfigurationException, IOException {
-        long totalDur = tlr.getTotalDuration();
-        Assert.assertTrue(totalDur > 0);
-        
-        Datagram[] ds = tlr.getDatagrams(totalDur - 1, 1);
-        Assert.assertTrue(ds != null);
-        Assert.assertTrue(ds.length == 1);
+	@Test
+	public void gotoTime2() throws IOException {
+		// setup
+		final int testIdx = NUMDATAGRAMS / 2;
+		long onTime = getTimeOfIndex(testIdx);
+		long afterTime = onTime + origDatagrams[testIdx].getDuration();
+		long midTime = onTime + ((afterTime - onTime) / 2);
+		// exercise
+		ByteBuffer bb = tlr.getByteBufferAtTime(midTime).getFirst();
+		Datagram d = tlr.getNextDatagram(bb);
+		// verify
+		assertEquals(origDatagrams[testIdx], d);
+	}
 
-        ds = tlr.getDatagrams(totalDur - 1, 2);
-        Assert.assertTrue(ds != null);
-        Assert.assertTrue(ds.length == 1);
-    }
+	@Test
+	public void gotoTime3() throws IOException {
+		// setup
+		final int testIdx = NUMDATAGRAMS / 2;
+		long onTime = getTimeOfIndex(testIdx);
+		long afterTime = onTime + origDatagrams[testIdx].getDuration();
+		// exercise
+		ByteBuffer bb = tlr.getByteBufferAtTime(afterTime).getFirst();
+		Datagram d = tlr.getNextDatagram(bb);
+		// verify
+		assertEquals(origDatagrams[testIdx + 1], d);
+	}
 
-    @Test
-    public void cannotGetAfterLastDatagram() throws MaryConfigurationException, IOException {
-        long totalDur = tlr.getTotalDuration();
-        Assert.assertTrue(totalDur > 0);
+	@Test
+	public void getDatagrams1() throws IOException {
+		// setup
+		final int testIdx = NUMDATAGRAMS / 2;
+		long onTime = getTimeOfIndex(testIdx);
+		Datagram[] D = null;
+		long span = origDatagrams[testIdx].getDuration();
+		long[] offset = new long[1];
+		// exercise
+		D = tlr.getDatagrams(onTime, span, sampleRate, offset);
+		// verify
+		assertEquals(1, D.length);
+		assertEquals(origDatagrams[testIdx], D[0]);
+		assertEquals(0l, offset[0]);
+	}
 
-        try {
-            Datagram d = tlr.getDatagram(totalDur);
-            Assert.fail("Should have thrown a BufferUnderflowException");
-        } catch (BufferUnderflowException e) {
-            // OK, expected
-        }
-    }
+	@Test
+	public void getDatagrams2() throws IOException {
+		// setup
+		final int testIdx = NUMDATAGRAMS / 2;
+		long onTime = getTimeOfIndex(testIdx);
+		Datagram[] D = null;
+		long span = origDatagrams[testIdx].getDuration() / 2;
+		long[] offset = new long[1];
+		// exercise
+		D = tlr.getDatagrams(onTime, span, sampleRate, offset);
+		// verify
+		assertEquals(1, D.length);
+		assertEquals(origDatagrams[testIdx], D[0]);
+		assertEquals(0l, offset[0]);
+	}
 
-    @Test
-    public void cannotGetAfterLastDatagrams() throws MaryConfigurationException, IOException {
-        long totalDur = tlr.getTotalDuration();
-        Assert.assertTrue(totalDur > 0);
+	@Test
+	public void getDatagrams3() throws IOException {
+		// setup
+		final int testIdx = NUMDATAGRAMS / 2;
+		long onTime = getTimeOfIndex(testIdx);
+		long afterTime = onTime + origDatagrams[testIdx].getDuration();
+		long midTime = onTime + ((afterTime - onTime) / 2);
+		Datagram[] D = null;
+		long span = origDatagrams[testIdx].getDuration() / 2;
+		// exercise
+		D = tlr.getDatagrams(midTime, span, sampleRate);
+		// verify
+		assertEquals(1, D.length);
+		assertEquals(origDatagrams[testIdx], D[0]);
+	}
 
-        try {
-            Datagram[] ds = tlr.getDatagrams(totalDur, 1);
-            Assert.fail("Should have thrown a BufferUnderflowException");
-        } catch (BufferUnderflowException e) {
-            // OK, expected
-        }
-    }
-    
-    
-    @Test
-    public void canReadLongDatagram() throws MaryConfigurationException, IOException {
-        // setup custom fixture for this method
-        TimelineReader timeline = new TimelineReader(tlFileName, false); // do not try memory mapping
-        // exercise
-        Datagram d = timeline.getDatagram(0);
-        // verify
-        Assert.assertEquals(origDatagrams[0].getLength(), d.getLength());
-    }
-    
-    
-    
-    @Test
-    public void canReadLongDatagrams1() throws MaryConfigurationException, IOException {
-        // setup custom fixture for this method
-        TimelineReader timeline = new TimelineReader(tlFileName, false); // do not try memory mapping
-        // exercise
-        Datagram[] ds = timeline.getDatagrams(0, origDatagrams[0].getDuration()+1);
-        // verify
-        Assert.assertEquals(2, ds.length);
-        Assert.assertEquals(origDatagrams[0].getLength(), ds[0].getLength());
-    }
-    
-    @Test
-    public void canReadLongDatagrams2() throws MaryConfigurationException, IOException {
-        // setup custom fixture for this method
-        TimelineReader timeline = new TimelineReader(tlFileName, false); // do not try memory mapping
-        // exercise
-        Datagram[] ds = timeline.getDatagrams(origDatagrams[0].getDuration(), origDatagrams[1].getDuration()+1);
-        // verify
-        Assert.assertEquals(2, ds.length);
-        Assert.assertEquals(origDatagrams[1].getLength(), ds[0].getLength());
-    }
-    
-    
-    
-    
-    
-    
-    
+	@Test
+	public void getDatagrams4() throws IOException {
+		// setup
+		final int testIdx = NUMDATAGRAMS / 2;
+		long onTime = getTimeOfIndex(testIdx);
+		Datagram[] D = null;
+		long span = origDatagrams[testIdx].getDuration() + 1;
+		long[] offset = new long[1];
+		// exercise
+		D = tlr.getDatagrams(onTime, span, sampleRate, offset);
+		// verify
+		assertEquals(2, D.length);
+		assertEquals(origDatagrams[testIdx], D[0]);
+		assertEquals(origDatagrams[testIdx + 1], D[1]);
+		assertEquals(0l, offset[0]);
+	}
 
-    @AfterClass
-    public static void tearDown() throws IOException {
-        /* Delete the test file */
-        File fid = new File( tlFileName );
-        fid.delete();
-        
-        /* ----------- */
-        System.out.println("End of the timeline symmetry test.");
-   }
+	@Test
+	public void getDatagrams5() throws IOException {
+		// setup
+		final int testIdx = NUMDATAGRAMS / 2;
+		long onTime = getTimeOfIndex(testIdx);
+		Datagram[] D = null;
+		long span = origDatagrams[testIdx].getDuration() + origDatagrams[testIdx + 1].getDuration();
+		long[] offset = new long[1];
+		// exercise
+		D = tlr.getDatagrams(onTime, span, sampleRate, offset);
+		// verify
+		assertEquals(2, D.length);
+		assertEquals(origDatagrams[testIdx], D[0]);
+		assertEquals(origDatagrams[testIdx + 1], D[1]);
+		assertEquals(0l, offset[0]);
+	}
+
+	@Test
+	public void getDatagrams6() throws IOException {
+		// setup
+		final int testIdx = NUMDATAGRAMS / 2;
+		long onTime = getTimeOfIndex(testIdx);
+		Datagram[] D = null;
+		long span = origDatagrams[testIdx].getDuration() + origDatagrams[testIdx + 1].getDuration();
+		long[] offset = new long[1];
+		// exercise
+		D = tlr.getDatagrams(onTime + 1, span, sampleRate, offset);
+		// verify
+		assertEquals("textIdx=" + testIdx + ", span=" + span + ", dur[" + testIdx + "]=" + origDatagrams[testIdx].getDuration()
+				+ ", dur[" + (testIdx + 1) + "]=" + origDatagrams[testIdx + 1].getDuration() + ", offset=" + offset[0], 3,
+				D.length);
+		assertEquals(origDatagrams[testIdx], D[0]);
+		assertEquals(origDatagrams[testIdx + 1], D[1]);
+		assertEquals(origDatagrams[testIdx + 2], D[2]);
+		assertEquals(1l, offset[0]);
+	}
+
+	@Test
+	public void getDatagrams7() throws IOException {
+		// setup
+		final int testIdx = NUMDATAGRAMS / 2;
+		long onTime = getTimeOfIndex(testIdx);
+		long afterTime = onTime + origDatagrams[testIdx].getDuration();
+		long midTime = onTime + ((afterTime - onTime) / 2);
+		Datagram[] D = null;
+		long dur = origDatagrams[testIdx].getDuration();
+		long span = dur - dur / 2 + 1;
+		long[] offset = new long[1];
+		// exercise
+		D = tlr.getDatagrams(midTime, span, sampleRate, offset);
+		// verify
+		assertEquals(2, D.length);
+		assertEquals(origDatagrams[testIdx], D[0]);
+		assertEquals(origDatagrams[testIdx + 1], D[1]);
+		assertEquals(dur / 2, offset[0]);
+	}
+
+	@Test
+	public void otherSampleRate() throws IOException {
+		// setup
+		final int testIdx = NUMDATAGRAMS / 2;
+		long onTime = getTimeOfIndex(testIdx);
+		Datagram[] D = null;
+		long dur = origDatagrams[testIdx].getDuration();
+		long span = dur;
+		// exercise
+		D = tlr.getDatagrams(onTime * 2, span * 2, sampleRate / 2);
+		// verify
+		assertEquals(1, D.length);
+		Assert.assertTrue(areEqual(D[0].getData(), origDatagrams[testIdx].getData()));
+		Assert.assertTrue(D[0].getDuration() != origDatagrams[testIdx].getDuration());
+
+	}
+
+	/**
+	 * @param testIdx
+	 * @return
+	 */
+	private long getTimeOfIndex(final int testIdx) {
+		long onTime = 0l;
+		for (int i = 0; i < testIdx; i++) {
+			onTime += origDatagrams[i].getDuration();
+		}
+		return onTime;
+	}
+
+	@Test
+	public void getLastDatagram() throws MaryConfigurationException, IOException {
+		long totalDur = tlr.getTotalDuration();
+		Assert.assertTrue(totalDur > 0);
+
+		Datagram d = tlr.getDatagram(totalDur - 1);
+		Assert.assertTrue(d != null);
+
+		long dur = d.getDuration();
+
+		d = tlr.getDatagram(totalDur - dur);
+		Assert.assertTrue(d != null);
+	}
+
+	@Test
+	public void getLastDatagrams() throws MaryConfigurationException, IOException {
+		long totalDur = tlr.getTotalDuration();
+		Assert.assertTrue(totalDur > 0);
+
+		Datagram[] ds = tlr.getDatagrams(totalDur - 1, 1);
+		Assert.assertTrue(ds != null);
+		Assert.assertTrue(ds.length == 1);
+
+		ds = tlr.getDatagrams(totalDur - 1, 2);
+		Assert.assertTrue(ds != null);
+		Assert.assertTrue(ds.length == 1);
+	}
+
+	@Test
+	public void cannotGetAfterLastDatagram() throws MaryConfigurationException, IOException {
+		long totalDur = tlr.getTotalDuration();
+		Assert.assertTrue(totalDur > 0);
+
+		try {
+			Datagram d = tlr.getDatagram(totalDur);
+			Assert.fail("Should have thrown a BufferUnderflowException");
+		} catch (BufferUnderflowException e) {
+			// OK, expected
+		}
+	}
+
+	@Test
+	public void cannotGetAfterLastDatagrams() throws MaryConfigurationException, IOException {
+		long totalDur = tlr.getTotalDuration();
+		Assert.assertTrue(totalDur > 0);
+
+		try {
+			Datagram[] ds = tlr.getDatagrams(totalDur, 1);
+			Assert.fail("Should have thrown a BufferUnderflowException");
+		} catch (BufferUnderflowException e) {
+			// OK, expected
+		}
+	}
+
+	@Test
+	public void canReadLongDatagram() throws MaryConfigurationException, IOException {
+		// setup custom fixture for this method
+		TimelineReader timeline = new TimelineReader(tlFileName, false); // do not try memory mapping
+		// exercise
+		Datagram d = timeline.getDatagram(0);
+		// verify
+		Assert.assertEquals(origDatagrams[0].getLength(), d.getLength());
+	}
+
+	@Test
+	public void canReadLongDatagrams1() throws MaryConfigurationException, IOException {
+		// setup custom fixture for this method
+		TimelineReader timeline = new TimelineReader(tlFileName, false); // do not try memory mapping
+		// exercise
+		Datagram[] ds = timeline.getDatagrams(0, origDatagrams[0].getDuration() + 1);
+		// verify
+		Assert.assertEquals(2, ds.length);
+		Assert.assertEquals(origDatagrams[0].getLength(), ds[0].getLength());
+	}
+
+	@Test
+	public void canReadLongDatagrams2() throws MaryConfigurationException, IOException {
+		// setup custom fixture for this method
+		TimelineReader timeline = new TimelineReader(tlFileName, false); // do not try memory mapping
+		// exercise
+		Datagram[] ds = timeline.getDatagrams(origDatagrams[0].getDuration(), origDatagrams[1].getDuration() + 1);
+		// verify
+		Assert.assertEquals(2, ds.length);
+		Assert.assertEquals(origDatagrams[1].getLength(), ds[0].getLength());
+	}
+
+	@AfterClass
+	public static void tearDown() throws IOException {
+		/* Delete the test file */
+		File fid = new File(tlFileName);
+		fid.delete();
+
+		/* ----------- */
+		System.out.println("End of the timeline symmetry test.");
+	}
+
+	/**
+	 * Compare two arrays of long.
+	 * 
+	 * @param a1
+	 *            an array of longs
+	 * @param a2
+	 *            an array of longs
+	 * @return true if they are equal, false if not.
+	 */
+	private static boolean areEqual(long[] a1, long[] a2) {
+		/* Check number of longs */
+		if (a1.length != a2.length)
+			return false;
+		/* Check contents */
+		for (int i = 0; i < a1.length; i++) {
+			if (a1[i] != a2[i])
+				return false;
+		}
+		/* If all passed, then the arrays are equal */
+		return (true);
+	}
+
+	/**
+	 * Compare two arrays of bytes.
+	 * 
+	 * @param a1
+	 *            an array of bytes
+	 * @param a2
+	 *            an array of bytes
+	 * @return true if they are equal, false if not.
+	 */
+	private static boolean areEqual(byte[] a1, byte[] a2) {
+		/* Check number of longs */
+		if (a1.length != a2.length)
+			return false;
+		/* Check contents */
+		for (int i = 0; i < a1.length; i++) {
+			if (a1[i] != a2[i])
+				return false;
+		}
+		/* If all passed, then the arrays are equal */
+		return (true);
+	}
 
-    
-    
-    
-    /**
-     * Compare two arrays of long.
-     * 
-     * @param a1 an array of longs
-     * @param a2 an array of longs
-     * @return true if they are equal, false if not.
-     */
-    private static boolean areEqual( long[] a1, long[] a2 ) {
-        /* Check number of longs */
-        if ( a1.length != a2.length ) return false;
-        /* Check contents */
-        for ( int i = 0; i < a1.length; i++ ) {
-            if ( a1[i] != a2[i] ) return false;
-        }
-        /* If all passed, then the arrays are equal */
-        return( true );
-    }
-    
-    /**
-     * Compare two arrays of bytes.
-     * 
-     * @param a1 an array of bytes
-     * @param a2 an array of bytes
-     * @return true if they are equal, false if not.
-     */
-    private static boolean areEqual( byte[] a1, byte[] a2 ) {
-        /* Check number of longs */
-        if ( a1.length != a2.length ) return false;
-        /* Check contents */
-        for ( int i = 0; i < a1.length; i++ ) {
-            if ( a1[i] != a2[i] ) return false;
-        }
-        /* If all passed, then the arrays are equal */
-        return( true );
-    }
-    
 }
-

marytts-client/src/main/java/marytts/client/MarySocketClient.java

@@ -46,475 +46,456 @@ import javax.sound.sampled.AudioFormat.Encoding;
 import marytts.util.http.Address;
 import marytts.util.string.StringUtils;
 
-
 /**
- * A socket client implementing the MARY protocol.
- * It can be used as a command line client or from within java code.
+ * A socket client implementing the MARY protocol. It can be used as a command line client or from within java code.
+ * 
  * @author Marc Schröder
  * @see MaryGUIClient A GUI interface to this client
  * @see marytts.server.MaryServer Description of the MARY protocol
  */
 
-public class MarySocketClient extends MaryClient
-{
+public class MarySocketClient extends MaryClient {
 
-    /**
-     * The simplest way to create a mary client. It will connect to the
-     * MARY server running at DFKI. Only use this for testing purposes!
-     * @throws IOException if communication with the server fails
-     * @throws UnknownHostException if the host could not be found
-     */
-    public MarySocketClient() throws IOException
-    {
-        super();
-    }
+	/**
+	 * The simplest way to create a mary client. It will connect to the MARY server running at DFKI. Only use this for testing
+	 * purposes!
+	 * 
+	 * @throws IOException
+	 *             if communication with the server fails
+	 * @throws UnknownHostException
+	 *             if the host could not be found
+	 */
+	public MarySocketClient() throws IOException {
+		super();
+	}
 
-    /**
-     * The typical way to create a mary client. It will connect to the
-     * MARY client running at the given host and port.
-     * This constructor reads two system properties:
-     * <ul>
-     * <li><code>mary.client.profile</code> (=true/false) - 
-     * determines whether profiling (timing) information is calculated;</li>
-     * <li><code>mary.client.quiet</code> (=true/false) - 
-     * tells the client not to print any of the normal information to stderr.</li>
-     * </ul>
-     * @param host the host where to contact a MARY server
-     * @param port the socket port where to contact a MARY server
-     * @throws IOException if communication with the server fails
-     * @throws UnknownHostException if the host could not be found
-     */
-    public MarySocketClient(Address serverAddress) throws IOException
-    {
-        super(serverAddress);
-    }
-    /**
-     * An alternative way to create a mary client, which works with applets.
-     * It will connect to the MARY client running at the given host and port.
-     * Note that in applets, the host must be the same as the one from which
-     * the applet was loaded; otherwise, a security exception is thrown.
-     * @param host the host where to contact a MARY server
-     * @param port the socket port where to contact a MARY server
-     * @param profile determines whether profiling (timing) information is calculated
-     * @param quiet tells the client not to print any of the normal information to stderr
-     * @throws IOException if communication with the server fails
-     * @throws UnknownHostException if the host could not be found
-     */
-    public MarySocketClient(Address serverAddress, boolean profile, boolean quiet) throws IOException
-    {
-        super(serverAddress, profile, quiet);
-    }
+	/**
+	 * The typical way to create a mary client. It will connect to the MARY client running at the given host and port. This
+	 * constructor reads two system properties:
+	 * <ul>
+	 * <li><code>mary.client.profile</code> (=true/false) - determines whether profiling (timing) information is calculated;</li>
+	 * <li><code>mary.client.quiet</code> (=true/false) - tells the client not to print any of the normal information to stderr.</li>
+	 * </ul>
+	 * 
+	 * @param host
+	 *            the host where to contact a MARY server
+	 * @param port
+	 *            the socket port where to contact a MARY server
+	 * @throws IOException
+	 *             if communication with the server fails
+	 * @throws UnknownHostException
+	 *             if the host could not be found
+	 */
+	public MarySocketClient(Address serverAddress) throws IOException {
+		super(serverAddress);
+	}
 
-    
+	/**
+	 * An alternative way to create a mary client, which works with applets. It will connect to the MARY client running at the
+	 * given host and port. Note that in applets, the host must be the same as the one from which the applet was loaded;
+	 * otherwise, a security exception is thrown.
+	 * 
+	 * @param host
+	 *            the host where to contact a MARY server
+	 * @param port
+	 *            the socket port where to contact a MARY server
+	 * @param profile
+	 *            determines whether profiling (timing) information is calculated
+	 * @param quiet
+	 *            tells the client not to print any of the normal information to stderr
+	 * @throws IOException
+	 *             if communication with the server fails
+	 * @throws UnknownHostException
+	 *             if the host could not be found
+	 */
+	public MarySocketClient(Address serverAddress, boolean profile, boolean quiet) throws IOException {
+		super(serverAddress, profile, quiet);
+	}
 
-    
-    @Override
-    protected void _process(String input, String inputType, String outputType, String locale, String audioType, 
-            String defaultVoiceName, String defaultStyle, String defaultEffects, 
-            Object output, long timeout, boolean streamingAudio, String outputTypeParams, AudioPlayerListener playerListener)
-        throws IOException
-    {
-        boolean isMaryAudioPlayer = false;
-        if (output instanceof marytts.util.data.audio.AudioPlayer) {
-            isMaryAudioPlayer = true;
-        } else if (output instanceof OutputStream) {
-        } else {
-            throw new IllegalArgumentException("Expected OutputStream or AudioPlayer, got " + output.getClass().getName());
-        }
-        final long startTime = System.currentTimeMillis();
-        // Socket Client
-        final Socket maryInfoSocket;
-        try {
-            maryInfoSocket = new Socket(data.hostAddress.getHost(), data.hostAddress.getPort());
-        } catch (SocketException se) {
-            throw new RuntimeException("Cannot connect to " + data.hostAddress.getFullAddress(), se);
-        }
-        final PrintWriter toServerInfo = new PrintWriter(new OutputStreamWriter(maryInfoSocket.getOutputStream(), "UTF-8"), true);
-        final BufferedReader fromServerInfo = new BufferedReader(new InputStreamReader(maryInfoSocket.getInputStream(), "UTF-8"));
+	@Override
+	protected void _process(String input, String inputType, String outputType, String locale, String audioType,
+			String defaultVoiceName, String defaultStyle, String defaultEffects, Object output, long timeout,
+			boolean streamingAudio, String outputTypeParams, AudioPlayerListener playerListener) throws IOException {
+		boolean isMaryAudioPlayer = false;
+		if (output instanceof marytts.util.data.audio.AudioPlayer) {
+			isMaryAudioPlayer = true;
+		} else if (output instanceof OutputStream) {
+		} else {
+			throw new IllegalArgumentException("Expected OutputStream or AudioPlayer, got " + output.getClass().getName());
+		}
+		final long startTime = System.currentTimeMillis();
+		// Socket Client
+		final Socket maryInfoSocket;
+		try {
+			maryInfoSocket = new Socket(data.hostAddress.getHost(), data.hostAddress.getPort());
+		} catch (SocketException se) {
+			throw new RuntimeException("Cannot connect to " + data.hostAddress.getFullAddress(), se);
+		}
+		final PrintWriter toServerInfo = new PrintWriter(new OutputStreamWriter(maryInfoSocket.getOutputStream(), "UTF-8"), true);
+		final BufferedReader fromServerInfo = new BufferedReader(new InputStreamReader(maryInfoSocket.getInputStream(), "UTF-8"));
 
-        // Formulate Request to Server:
-        //System.err.println("Writing request to server.");
-        toServerInfo.print("MARY IN=" + inputType + " OUT=" + outputType + " LOCALE="+ locale);
-        if (audioType != null) {
-        	if (streamingAudio && data.serverCanStream) {
-        		toServerInfo.print(" AUDIO=STREAMING_"+audioType);
-        	} else {
-        		toServerInfo.print(" AUDIO=" + audioType);
-        	}
-        }
-        if (defaultVoiceName != null && !defaultVoiceName.equals("")) {
-            toServerInfo.print(" VOICE=" + defaultVoiceName);
-        }
-        
-        if (defaultStyle != null && !defaultStyle.equals("")) {
-            toServerInfo.print(" STYLE=" + defaultStyle);
-        }
-        
-        if (defaultEffects != null && !defaultEffects.equals("")) {
-            toServerInfo.print(" EFFECTS=" + defaultEffects);
-        }
-        toServerInfo.println();
-        
-        // Receive a request ID:
-        //System.err.println("Reading reply from server.");
-        String helper = fromServerInfo.readLine();
-        //System.err.println("Read from Server: " + helper);
-        int id = -1;
-        try {
-            id = Integer.parseInt(helper);
-        } catch (NumberFormatException e) {
-            // Whatever we read from the server, it was not a number
-            StringBuilder message = new StringBuilder("Server replied:\n");
-            message.append(helper);
-            message.append("\n");
-            while ((helper = fromServerInfo.readLine()) != null) {
-                message.append(helper);
-                message.append("\n");
-            }
-            throw new IOException(message.toString());
-        }
+		// Formulate Request to Server:
+		// System.err.println("Writing request to server.");
+		toServerInfo.print("MARY IN=" + inputType + " OUT=" + outputType + " LOCALE=" + locale);
+		if (audioType != null) {
+			if (streamingAudio && data.serverCanStream) {
+				toServerInfo.print(" AUDIO=STREAMING_" + audioType);
+			} else {
+				toServerInfo.print(" AUDIO=" + audioType);
+			}
+		}
+		if (defaultVoiceName != null && !defaultVoiceName.equals("")) {
+			toServerInfo.print(" VOICE=" + defaultVoiceName);
+		}
 
-        //System.err.println("Read id " + id + " from server.");
-        final Socket maryDataSocket = new Socket(data.hostAddress.getHost(), data.hostAddress.getPort());
-        //System.err.println("Created second socket.");
-        final PrintWriter toServerData = new PrintWriter(new OutputStreamWriter(maryDataSocket.getOutputStream(), "UTF-8"), true);
-        toServerData.println(id);
-        //System.err.println("Writing to server:");
-        //System.err.print(input);
-        toServerData.println(input.trim());
-        maryDataSocket.shutdownOutput();
+		if (defaultStyle != null && !defaultStyle.equals("")) {
+			toServerInfo.print(" STYLE=" + defaultStyle);
+		}
 
-        // Check for warnings from server:
-        final WarningReader warningReader = new WarningReader(fromServerInfo);
-        warningReader.start();
-        
-        // Read from Server and copy into OutputStream output:
-        // (as we only do low-level copying of bytes here,
-        //  we do not need to distinguish between text and audio)
-        final InputStream fromServerStream = maryDataSocket.getInputStream();
+		if (defaultEffects != null && !defaultEffects.equals("")) {
+			toServerInfo.print(" EFFECTS=" + defaultEffects);
+		}
+		toServerInfo.println();
 
-        // If timeout is > 0, create a timer. It will close the input stream,
-        // thus causing an IOException in the reading code.
-        final Timer timer;
-        if (timeout <= 0) {
-            timer = null;
-        } else {
-            timer = new Timer();
-            TimerTask timerTask = new TimerTask() {
-                public void run() {
-                    System.err.println("Timer closes socket");
-                    try {
-                        maryDataSocket.close();
-                    } catch (IOException ioe) {
-                        ioe.printStackTrace();
-                    }
-                }
-            };
-            timer.schedule(timerTask, timeout);
-        }
+		// Receive a request ID:
+		// System.err.println("Reading reply from server.");
+		String helper = fromServerInfo.readLine();
+		// System.err.println("Read from Server: " + helper);
+		int id = -1;
+		try {
+			id = Integer.parseInt(helper);
+		} catch (NumberFormatException e) {
+			// Whatever we read from the server, it was not a number
+			StringBuilder message = new StringBuilder("Server replied:\n");
+			message.append(helper);
+			message.append("\n");
+			while ((helper = fromServerInfo.readLine()) != null) {
+				message.append(helper);
+				message.append("\n");
+			}
+			throw new IOException(message.toString());
+		}
 
-        if (isMaryAudioPlayer) {
-            final marytts.util.data.audio.AudioPlayer player = (marytts.util.data.audio.AudioPlayer) output;
-            final AudioPlayerListener listener = playerListener;
-            Thread t = new Thread() {
-                public void run() 
-                {
-                    try {
-                        InputStream in = fromServerStream;
-                        if (doProfile)
-                            System.err.println("After "+(System.currentTimeMillis()-startTime)+" ms: Trying to read data from server");
-                        in = new BufferedInputStream(in);
-                        in.mark(1000);
-                         AudioInputStream fromServerAudio = AudioSystem.getAudioInputStream(in);
-                         if (fromServerAudio.getFrameLength() == 0) { // weird bug under Java 1.4
-                             //in.reset();
-                             fromServerAudio = new AudioInputStream(in, fromServerAudio.getFormat(), AudioSystem.NOT_SPECIFIED);
-                         }
-                         //System.out.println("Audio framelength: "+fromServerAudio.getFrameLength());
-                         //System.out.println("Audio frame size: "+fromServerAudio.getFormat().getFrameSize());
-                         //System.out.println("Audio format: "+fromServerAudio.getFormat());
-                         if (doProfile)
-                             System.err.println("After "+(System.currentTimeMillis()-startTime)+" ms: Audio available: "+in.available());
-                         AudioFormat audioFormat = fromServerAudio.getFormat();
-                         if (!audioFormat.getEncoding().equals(Encoding.PCM_SIGNED)) { // need conversion, e.g. for mp3
-                             audioFormat = new AudioFormat(fromServerAudio.getFormat().getSampleRate(), 16, 1, true, false);
-                             fromServerAudio = AudioSystem.getAudioInputStream(audioFormat, fromServerAudio);
-                         }
-                        player.setAudio(fromServerAudio);
-                        player.run(); // not start(), i.e. execute in this thread
-                        if (timer != null) {
-                            timer.cancel();
-                        }
-                        if (listener != null) listener.playerFinished();
+		// System.err.println("Read id " + id + " from server.");
+		final Socket maryDataSocket = new Socket(data.hostAddress.getHost(), data.hostAddress.getPort());
+		// System.err.println("Created second socket.");
+		final PrintWriter toServerData = new PrintWriter(new OutputStreamWriter(maryDataSocket.getOutputStream(), "UTF-8"), true);
+		toServerData.println(id);
+		// System.err.println("Writing to server:");
+		// System.err.print(input);
+		toServerData.println(input.trim());
+		maryDataSocket.shutdownOutput();
 
-                        toServerInfo.close();
-                        fromServerInfo.close();
-                        maryInfoSocket.close();
-                        toServerData.close();
-                        maryDataSocket.close();
-                    } catch (Exception e) {
-                        e.printStackTrace();
-                    }                    
+		// Check for warnings from server:
+		final WarningReader warningReader = new WarningReader(fromServerInfo);
+		warningReader.start();
 
-                    try {
-                        warningReader.join();
-                    } catch (InterruptedException ie) {}
-                    if (warningReader.getWarnings().length() > 0) { // there are warnings
-                        String warnings = warningReader.getWarnings(); 
-                        System.err.println(warnings);
-                        if (listener != null) listener.playerException(new IOException(warnings));
-                    }
+		// Read from Server and copy into OutputStream output:
+		// (as we only do low-level copying of bytes here,
+		// we do not need to distinguish between text and audio)
+		final InputStream fromServerStream = maryDataSocket.getInputStream();
 
-                        if (doProfile) {
-                            long endTime = System.currentTimeMillis();
-                            long processingTime = endTime - startTime;
-                            System.err.println("Processed request in " + processingTime + " ms.");
-                        }
-                }
-            };
-            if (streamingAudio) {
-                t.start();
-            } else {
-                t.run(); // execute code in the current thread
-            }
-        } else { // output is an OutputStream
-            OutputStream os = (OutputStream) output;
-            InputStream bis = new BufferedInputStream(fromServerStream);
-            byte[] bbuf = new byte[1024];
-            int nr;
-            while ((nr = bis.read(bbuf, 0, bbuf.length)) != -1) {
-                //System.err.println("Read " + nr + " bytes from server.");
-                os.write(bbuf, 0, nr);
-            }
-            os.flush();
-            
+		// If timeout is > 0, create a timer. It will close the input stream,
+		// thus causing an IOException in the reading code.
+		final Timer timer;
+		if (timeout <= 0) {
+			timer = null;
+		} else {
+			timer = new Timer();
+			TimerTask timerTask = new TimerTask() {
+				public void run() {
+					System.err.println("Timer closes socket");
+					try {
+						maryDataSocket.close();
+					} catch (IOException ioe) {
+						ioe.printStackTrace();
+					}
+				}
+			};
+			timer.schedule(timerTask, timeout);
+		}
 
-            if (timeout > 0) {
-                timer.cancel();
-            }
+		if (isMaryAudioPlayer) {
+			final marytts.util.data.audio.AudioPlayer player = (marytts.util.data.audio.AudioPlayer) output;
+			final AudioPlayerListener listener = playerListener;
+			Thread t = new Thread() {
+				public void run() {
+					try {
+						InputStream in = fromServerStream;
+						if (doProfile)
+							System.err.println("After " + (System.currentTimeMillis() - startTime)
+									+ " ms: Trying to read data from server");
+						in = new BufferedInputStream(in);
+						in.mark(1000);
+						AudioInputStream fromServerAudio = AudioSystem.getAudioInputStream(in);
+						if (fromServerAudio.getFrameLength() == 0) { // weird bug under Java 1.4
+							// in.reset();
+							fromServerAudio = new AudioInputStream(in, fromServerAudio.getFormat(), AudioSystem.NOT_SPECIFIED);
+						}
+						// System.out.println("Audio framelength: "+fromServerAudio.getFrameLength());
+						// System.out.println("Audio frame size: "+fromServerAudio.getFormat().getFrameSize());
+						// System.out.println("Audio format: "+fromServerAudio.getFormat());
+						if (doProfile)
+							System.err.println("After " + (System.currentTimeMillis() - startTime) + " ms: Audio available: "
+									+ in.available());
+						AudioFormat audioFormat = fromServerAudio.getFormat();
+						if (!audioFormat.getEncoding().equals(Encoding.PCM_SIGNED)) { // need conversion, e.g. for mp3
+							audioFormat = new AudioFormat(fromServerAudio.getFormat().getSampleRate(), 16, 1, true, false);
+							fromServerAudio = AudioSystem.getAudioInputStream(audioFormat, fromServerAudio);
+						}
+						player.setAudio(fromServerAudio);
+						player.run(); // not start(), i.e. execute in this thread
+						if (timer != null) {
+							timer.cancel();
+						}
+						if (listener != null)
+							listener.playerFinished();
 
-            toServerInfo.close();
-            fromServerInfo.close();
-            maryInfoSocket.close();
-            toServerData.close();
-            maryDataSocket.close();
+						toServerInfo.close();
+						fromServerInfo.close();
+						maryInfoSocket.close();
+						toServerData.close();
+						maryDataSocket.close();
+					} catch (Exception e) {
+						e.printStackTrace();
+					}
 
-            try {
-                warningReader.join();
-            } catch (InterruptedException ie) {}
-            if (warningReader.getWarnings().length() > 0) // there are warnings
-                throw new IOException(warningReader.getWarnings());
+					try {
+						warningReader.join();
+					} catch (InterruptedException ie) {
+					}
+					if (warningReader.getWarnings().length() > 0) { // there are warnings
+						String warnings = warningReader.getWarnings();
+						System.err.println(warnings);
+						if (listener != null)
+							listener.playerException(new IOException(warnings));
+					}
 
-            if (doProfile) {
-                long endTime = System.currentTimeMillis();
-                long processingTime = endTime - startTime;
-                System.err.println("Processed request in " + processingTime + " ms.");
-            }
-        }
-    }
+					if (doProfile) {
+						long endTime = System.currentTimeMillis();
+						long processingTime = endTime - startTime;
+						System.err.println("Processed request in " + processingTime + " ms.");
+					}
+				}
+			};
+			if (streamingAudio) {
+				t.start();
+			} else {
+				t.run(); // execute code in the current thread
+			}
+		} else { // output is an OutputStream
+			OutputStream os = (OutputStream) output;
+			InputStream bis = new BufferedInputStream(fromServerStream);
+			byte[] bbuf = new byte[1024];
+			int nr;
+			while ((nr = bis.read(bbuf, 0, bbuf.length)) != -1) {
+				// System.err.println("Read " + nr + " bytes from server.");
+				os.write(bbuf, 0, nr);
+			}
+			os.flush();
 
-    
-    /**
-     * From an open server connection, read one chunk of info data. Writes the 
-     * infoCommand to the server, then reads from the server until an empty line
-     * or eof is read.
-     * @param toServer
-     * @param fromServer
-     * @param infoCommand the one-line request to send to the server
-     * @return a string representing the server response, lines being separated by a '\n' character.
-     * @throws IOException if communication with the server fails
-     */
-    private String getServerInfo(String infoCommand)
-    throws IOException
-    {
-        Socket marySocket = new Socket(data.hostAddress.getHost(), data.hostAddress.getPort());
-        PrintWriter toServerInfo = new PrintWriter(new OutputStreamWriter(marySocket.getOutputStream(), "UTF-8"), true);
-        BufferedReader fromServerInfo = new BufferedReader(new InputStreamReader(marySocket.getInputStream(), "UTF-8"));
+			if (timeout > 0) {
+				timer.cancel();
+			}
 
-        toServerInfo.println(infoCommand);
-        StringBuilder result = new StringBuilder();
-        String line = null;
-        // Read until either end of file or an empty line
-        while((line = fromServerInfo.readLine()) != null && !line.equals("")) {
-            result.append(line);
-            result.append("\n");
-        }
-        marySocket.close();
-        return result.toString();
-    }
+			toServerInfo.close();
+			fromServerInfo.close();
+			maryInfoSocket.close();
+			toServerData.close();
+			maryDataSocket.close();
 
+			try {
+				warningReader.join();
+			} catch (InterruptedException ie) {
+			}
+			if (warningReader.getWarnings().length() > 0) // there are warnings
+				throw new IOException(warningReader.getWarnings());
 
+			if (doProfile) {
+				long endTime = System.currentTimeMillis();
+				long processingTime = endTime - startTime;
+				System.err.println("Processed request in " + processingTime + " ms.");
+			}
+		}
+	}
 
-    /**
-     * Get the version info from the server. This is optional information
-     * which is not required for the normal operation of the client, but
-     * may help to avoid incompatibilities.
-     * @return a string array, each String representing one line of info
-     * without the trailing newline character.
-     * @throws IOException if communication with the server fails
-     * @throws UnknownHostException if the host could not be found
-     */
-    protected void fillServerVersion() throws IOException, UnknownHostException
-    {
-        // Expect 3 lines of the kind
-        // Mary TTS server
-        // Specification version 1.9.1
-        // Implementation version 20030207
-        String info = getServerInfo("MARY VERSION");
-        if (info.length() == 0)
-            throw new IOException("Could not get version info from Mary server");
-        info = info.replace('\n', ' ');
-        data.toServerVersionInfo(info);
-    }
-    
-    
+	/**
+	 * From an open server connection, read one chunk of info data. Writes the infoCommand to the server, then reads from the
+	 * server until an empty line or eof is read.
+	 * 
+	 * @param toServer
+	 * @param fromServer
+	 * @param infoCommand
+	 *            the one-line request to send to the server
+	 * @return a string representing the server response, lines being separated by a '\n' character.
+	 * @throws IOException
+	 *             if communication with the server fails
+	 */
+	private String getServerInfo(String infoCommand) throws IOException {
+		Socket marySocket = new Socket(data.hostAddress.getHost(), data.hostAddress.getPort());
+		PrintWriter toServerInfo = new PrintWriter(new OutputStreamWriter(marySocket.getOutputStream(), "UTF-8"), true);
+		BufferedReader fromServerInfo = new BufferedReader(new InputStreamReader(marySocket.getInputStream(), "UTF-8"));
 
+		toServerInfo.println(infoCommand);
+		StringBuilder result = new StringBuilder();
+		String line = null;
+		// Read until either end of file or an empty line
+		while ((line = fromServerInfo.readLine()) != null && !line.equals("")) {
+			result.append(line);
+			result.append("\n");
+		}
+		marySocket.close();
+		return result.toString();
+	}
 
+	/**
+	 * Get the version info from the server. This is optional information which is not required for the normal operation of the
+	 * client, but may help to avoid incompatibilities.
+	 * 
+	 * @return a string array, each String representing one line of info without the trailing newline character.
+	 * @throws IOException
+	 *             if communication with the server fails
+	 * @throws UnknownHostException
+	 *             if the host could not be found
+	 */
+	protected void fillServerVersion() throws IOException, UnknownHostException {
+		// Expect 3 lines of the kind
+		// Mary TTS server
+		// Specification version 1.9.1
+		// Implementation version 20030207
+		String info = getServerInfo("MARY VERSION");
+		if (info.length() == 0)
+			throw new IOException("Could not get version info from Mary server");
+		info = info.replace('\n', ' ');
+		data.toServerVersionInfo(info);
+	}
 
-    @Override
-    protected void fillDataTypes() throws UnknownHostException, IOException
-    {
-        // Expect a variable number of lines of the kind
-        // RAWMARYXML INPUT OUTPUT
-        // TEXT_DE LOCALE=de INPUT
-        // AUDIO OUTPUT
-        String info = getServerInfo("MARY LIST DATATYPES");
-        if (info.length() == 0)
-            throw new IOException("Could not get list of data types from Mary server");
-        data.toDataTypes(info);
-    }
+	@Override
+	protected void fillDataTypes() throws UnknownHostException, IOException {
+		// Expect a variable number of lines of the kind
+		// RAWMARYXML INPUT OUTPUT
+		// TEXT_DE LOCALE=de INPUT
+		// AUDIO OUTPUT
+		String info = getServerInfo("MARY LIST DATATYPES");
+		if (info.length() == 0)
+			throw new IOException("Could not get list of data types from Mary server");
+		data.toDataTypes(info);
+	}
 
+	@Override
+	protected void fillVoices() throws IOException, UnknownHostException {
+		// Expect a variable number of lines of the kind
+		// de7 de female
+		// us2 en male
+		// dfki-stadium-emo de male limited
+		String info = getServerInfo("MARY LIST VOICES");
+		if (info.length() == 0)
+			throw new IOException("Could not get voice list from Mary server");
+		data.toVoices(info);
+	}
 
-    @Override
-    protected void fillVoices() throws IOException, UnknownHostException
-    {
-        // Expect a variable number of lines of the kind
-        // de7 de female
-        // us2 en male
-        // dfki-stadium-emo de male limited
-        String info = getServerInfo("MARY LIST VOICES");
-        if (info.length() == 0)
-            throw new IOException("Could not get voice list from Mary server");
-        data.toVoices(info);
-    }
+	@Override
+	protected void fillLocales() throws IOException, UnknownHostException {
+		String info = getServerInfo("MARY LIST LOCALES");
+		if (info.length() == 0)
+			throw new IOException("Could not get locales list from Mary server");
+		data.toLocales(info);
+	}
 
-    @Override
-    protected void fillLocales() throws IOException, UnknownHostException
-    {
-        String info = getServerInfo("MARY LIST LOCALES");
-        if (info.length() == 0)
-            throw new IOException("Could not get locales list from Mary server");
-        data.toLocales(info);
-    }
+	@Override
+	protected void fillVoiceExampleTexts(String voicename) throws IOException {
+		String info = getServerInfo("MARY VOICE EXAMPLETEXT " + voicename);
+		if (info.length() == 0)
+			throw new IOException("Could not get example text from Mary server");
 
-    @Override
-    protected void fillVoiceExampleTexts(String voicename) throws IOException
-    {
-        String info = getServerInfo("MARY VOICE EXAMPLETEXT " + voicename);
-        if (info.length() == 0)
-            throw new IOException("Could not get example text from Mary server");
-        
-        StringTokenizer st = new StringTokenizer(info,"\n");
-        Vector<String> sentences = new Vector<String>();
-        while (st.hasMoreTokens()) {
-            sentences.add(st.nextToken());
-        }
-        data.voiceExampleTextsLimitedDomain.put(voicename, sentences);
-    }
+		StringTokenizer st = new StringTokenizer(info, "\n");
+		Vector<String> sentences = new Vector<String>();
+		while (st.hasMoreTokens()) {
+			sentences.add(st.nextToken());
+		}
+		data.voiceExampleTextsLimitedDomain.put(voicename, sentences);
+	}
 
-    
-    @Override
-    protected void fillServerExampleText(String dataType, String locale) throws IOException
-    {
-        String info = getServerInfo("MARY EXAMPLETEXT " + dataType + " " + locale);
-        
-        if (info.length() == 0)
-            throw new IOException("Could not get example text from Mary server");
-        
-        data.serverExampleTexts.put(dataType+" "+locale, info.replaceAll("\n", System.getProperty("line.separator")));
-    }
+	@Override
+	protected void fillServerExampleText(String dataType, String locale) throws IOException {
+		String info = getServerInfo("MARY EXAMPLETEXT " + dataType + " " + locale);
 
-    /**
-     * Request the available audio effects for a voice from the server
-     * @param voicename the voice
-     * @return A string of available audio effects and default parameters, i.e. "FIRFilter,Robot(amount=50)"
-     * @throws IOException
-     * @throws UnknownHostException
-     */
-    @Override
-    protected String getDefaultAudioEffects() throws IOException
-    {
-        return getServerInfo("MARY VOICE GETDEFAULTAUDIOEFFECTS");
-    }
-    
-    
-    @Override
-    public String requestDefaultEffectParameters(String effectName) throws IOException, UnknownHostException 
-    {
-        String info = getServerInfo("MARY VOICE GETAUDIOEFFECTDEFAULTPARAM " + effectName);
-        return info.replaceAll("\n", System.getProperty("line.separator"));
-    }
+		if (info.length() == 0)
+			throw new IOException("Could not get example text from Mary server");
 
-    @Override
-    public String requestFullEffect(String effectName, String currentEffectParams) throws IOException, UnknownHostException 
-    {
-        String info = getServerInfo("MARY VOICE GETFULLAUDIOEFFECT " + effectName + " " + currentEffectParams);
-        return info.replaceAll("\n", System.getProperty("line.separator"));
-    }
-    
-    @Override
-    protected void fillEffectHelpText(String effectName) throws IOException 
-    {
-        String info = getServerInfo("MARY VOICE GETAUDIOEFFECTHELPTEXT " + effectName);
-        data.audioEffectHelpTextsMap.put(effectName, info.replaceAll("\n", System.getProperty("line.separator")));
-    }
-    
-    @Override
-    public boolean isHMMEffect(String effectName) throws IOException, UnknownHostException
-    {
-        String info = getServerInfo("MARY VOICE ISHMMAUDIOEFFECT " + effectName);
-        
-        if (info.length() == 0)
-            return false;
+		data.serverExampleTexts.put(dataType + " " + locale, info.replaceAll("\n", System.getProperty("line.separator")));
+	}
 
-        boolean bRet = false;
-        info = info.toLowerCase();
-        if (info.indexOf("yes")>-1)
-            bRet = true;
-        
-        return bRet;
-    }
-    
-    public String getFeatures(String locale) throws IOException
-    {
-        throw new RuntimeException("not implemented");
-    }
-    
-    public String getFeaturesForVoice(String voice) throws IOException
-    {
-        throw new RuntimeException("not implemented");
-    }
+	/**
+	 * Request the available audio effects for a voice from the server
+	 * 
+	 * @param voicename
+	 *            the voice
+	 * @return A string of available audio effects and default parameters, i.e. "FIRFilter,Robot(amount=50)"
+	 * @throws IOException
+	 * @throws UnknownHostException
+	 */
+	@Override
+	protected String getDefaultAudioEffects() throws IOException {
+		return getServerInfo("MARY VOICE GETDEFAULTAUDIOEFFECTS");
+	}
 
-    
-    
-    @Override
-    protected void fillAudioFileFormatAndOutTypes()  throws IOException
-    {
-        String audioFormatInfo =  getServerInfo("MARY LIST AUDIOFILEFORMATTYPES");
-        data.audioOutTypes = new Vector<String>(Arrays.asList(StringUtils.toStringArray(audioFormatInfo)));
-        data.audioFileFormatTypes = new Vector<String>();
-        for (String af : data.audioOutTypes) {
-            if (af.endsWith("_FILE")) {
-                String typeName = af.substring(0, af.indexOf("_"));
-                try {
-                    AudioFileFormat.Type type = MaryClient.getAudioFileFormatType(typeName);
-                    data.audioFileFormatTypes.add(typeName+" "+type.getExtension());
-                } catch (Exception e) {}
-            }
-        }
-    }
+	@Override
+	public String requestDefaultEffectParameters(String effectName) throws IOException, UnknownHostException {
+		String info = getServerInfo("MARY VOICE GETAUDIOEFFECTDEFAULTPARAM " + effectName);
+		return info.replaceAll("\n", System.getProperty("line.separator"));
+	}
 
+	@Override
+	public String requestFullEffect(String effectName, String currentEffectParams) throws IOException, UnknownHostException {
+		String info = getServerInfo("MARY VOICE GETFULLAUDIOEFFECT " + effectName + " " + currentEffectParams);
+		return info.replaceAll("\n", System.getProperty("line.separator"));
+	}
 
+	@Override
+	protected void fillEffectHelpText(String effectName) throws IOException {
+		String info = getServerInfo("MARY VOICE GETAUDIOEFFECTHELPTEXT " + effectName);
+		data.audioEffectHelpTextsMap.put(effectName, info.replaceAll("\n", System.getProperty("line.separator")));
+	}
+
+	@Override
+	public boolean isHMMEffect(String effectName) throws IOException, UnknownHostException {
+		String info = getServerInfo("MARY VOICE ISHMMAUDIOEFFECT " + effectName);
+
+		if (info.length() == 0)
+			return false;
+
+		boolean bRet = false;
+		info = info.toLowerCase();
+		if (info.indexOf("yes") > -1)
+			bRet = true;
+
+		return bRet;
+	}
+
+	public String getFeatures(String locale) throws IOException {
+		throw new RuntimeException("not implemented");
+	}
+
+	public String getFeaturesForVoice(String voice) throws IOException {
+		throw new RuntimeException("not implemented");
+	}
+
+	@Override
+	protected void fillAudioFileFormatAndOutTypes() throws IOException {
+		String audioFormatInfo = getServerInfo("MARY LIST AUDIOFILEFORMATTYPES");
+		data.audioOutTypes = new Vector<String>(Arrays.asList(StringUtils.toStringArray(audioFormatInfo)));
+		data.audioFileFormatTypes = new Vector<String>();
+		for (String af : data.audioOutTypes) {
+			if (af.endsWith("_FILE")) {
+				String typeName = af.substring(0, af.indexOf("_"));
+				try {
+					AudioFileFormat.Type type = MaryClient.getAudioFileFormatType(typeName);
+					data.audioFileFormatTypes.add(typeName + " " + type.getExtension());
+				} catch (Exception e) {
+				}
+			}
+		}
+	}
 
 }
-