root/Trunk/Scripts/AcousticModel_scripts/HTK/AMCreate_scripts/Compile_AcousticModel.sh

#!/bin/sh
####################################################################
###
### script name: Compile_AcouticModel.sh
### modified by: Ken MacLean
### email: contact@voxforge.org
### Date: 2006.06.16
### Command: ./Compile_AcousticModel.sh
### Version: 0.1
###             
### Copyright (C) 2006 Ken MacLean
###
### This program is free software; you can redistribute it and/or
### modify it under the terms of the GNU General Public License
### as published by the Free Software Foundation; either version 3
### of the License, or (at your option) any later version.
###
### This program is distributed in the hope that it will be useful,
### but WITHOUT ANY WARRANTY; without even the implied warranty of
### MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
### GNU General Public License for more details.
###
####################################################################
### Input Files
####################################################################
# update this files:
#    master_prompts file needs directory paths since train.scp is derived from this file!

####################################################################
### Subroutines
####################################################################
htk_init () {
        cd ./interim_files
        #create directories if they do not already exist
        mkdir hmm0 hmm1 hmm2 hmm3 hmm4 hmm5 hmm6 hmm7 hmm8 hmm9 hmm10 hmm11 hmm12 hmm13 hmm14 hmm15
        #all these rms were to allow the contents of the interim directory to be committed to svn
    rm -f hmm0/hmmdefs hmm0/macros hmm0/proto hmm0/vFloors
    rm -f hmm1/hmmdefs hmm1/macros
    rm -f hmm2/hmmdefs hmm2/macros
    rm -f hmm3/hmmdefs hmm3/hmmdefs.tmp2  hmm3/macros
    rm -f hmm4/hmmdefs hmm4/hmmdefs.tmp2  hmm4/macros
    rm -f hmm5/hmmdefs hmm5/macros
    rm -f hmm6/hmmdefs hmm6/macros   
    rm -f hmm7/hmmdefs hmm7/macros
    rm -f hmm8/hmmdefs hmm8/macros
    rm -f hmm9/hmmdefs hmm9/macros
    rm -f hmm10/hmmdefs hmm10/macros
    rm -f hmm11/hmmdefs hmm11/macros
    rm -f hmm12/hmmdefs hmm12/macros  
    rm -f hmm13/hmmdefs hmm13/macros
    rm -f hmm14/hmmdefs hmm14/macros
    rm -f hmm15/hmmdefs hmm15/macros    
        rm -f ./aligned.mlf
        rm -f ./dict
        rm -f ./dict1
        rm -f ./dict-tri
        rm -f ./fulllist
        rm -f ./fulllist-original
#       rm -f ./master_prompts don't delete - created by fix_master_prompts.pl
        rm -f ./mktri.hed
        rm -f ./monophones0
        rm -f ./monophones1
        rm -f ./phones0.mlf
        rm -f ./phones1.mlf
        rm -f ./stats
        rm -f ./tiedlist
        rm -f ./train.scp
        rm -f ./tree.hed
        rm -f ./trees
        rm -f ./triphones1
        rm -f ./wintri.mlf
        rm -f ./wlist
        rm -f ./words.mlf
    cd ..
    rm -f ../acoustic_model_files/hmmdefs
    rm -f ../acoustic_model_files/tiedlist
    rm -f ../acoustic_model_files/macros
    cd logs
  
    rm -f ./Step2_HDMan1_log
    rm -f ./Step2_HDMan2_log
    rm -f ./Step4_HLEd_phones0_log
    rm -f ./Step4_HLEd_phones1_log
    rm -f ./Step6_HCompV_hmm0_log
    rm -f ./Step6_HERest_hmm1_log
    rm -f ./Step6_HERest_hmm2_log
    rm -f ./Step6_HERest_hmm3_log
    rm -f ./Step7_HERest_hmm6_log
    rm -f ./Step7_HERest_hmm7_log
    rm -f ./Step7_HHEd_hmm5_log
    rm -f ./Step8_HERest_hmm8_log
    rm -f ./Step8_HERest_hmm9_log
    rm -f ./Step8_HVite_log
    rm -f ./Step9_HERest_hmm10_log
    rm -f ./Step9_HERest_hmm11_log
    rm -f ./Step9_HLed_log
    rm -f ./Step9_hmm10_HHEd_log
    rm -f ./Step10_HDMan_flog
    rm -f ./Step10_HDMan_log
    rm -f ./Step10_HHed_hmm13_log
    rm -f ./Step10_HERest_hmm14_log
    rm -f ./Step10_HERest_hmm15_log
    
    cd ..
        return 0        
}

make_wlist_label_file () {
perl $HTK/prompts2wlist ./interim_files/master_prompts ./interim_files/wlist

    echo 'SENT-END' >> ./interim_files/wlist 
        echo 'SENT-START' >> ./interim_files/wlist
        perl ./perlsort.pl  ./interim_files/wlist ./interim_files/wlist1
        uniq ./interim_files/wlist1 ./interim_files/wlist
    rm ./interim_files/wlist1
        return 0
}

make_monophones0 () {
        for STR in `cat ./interim_files/monophones1`; # monophones1 = monophones0 less "sp" phone
        do 
                if [ "${STR}" != "sp" ]; then 
                        echo ${STR} >> ./interim_files/monophones0; 
                fi; 
        done;
        return 0
}

make_wordsmlf () {
        perl $HTK/prompts2mlf ./interim_files/words.mlf ./interim_files/master_prompts


        return 0
}

make_hmmdefs () {
        for WORD in `cat ./interim_files/monophones0`
        do 
                tail -n 28  ./interim_files/hmm0/proto | sed s/~h\ \"proto\"/~h\ \"$WORD\"/g >> ./interim_files/hmm0/hmmdefs
        done                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        
        return 0
}

make_macros () {
        head -n 3 ./interim_files/hmm0/proto > ./interim_files/hmm0/macros
        cat ./interim_files/hmm0/vFloors >> ./interim_files/hmm0/macros
        return 0
}

make_hmm4 () {
    LINE="start"
    NUM=1

        rm -rf ./interim_files/hmm4/hmmdefs.tmp1
        rm -rf ./interim_files/hmm4/hmmdefs.tmp2
        
    while [ `echo ${LINE} | awk '{ print $1 }'` != "" ];
    do
                LINE=`cat -n ./interim_files/hmm4/hmmdefs | grep ^[[:space:]]*${NUM}[[:space:]] | sed s/^[[:space:]]*[[:digit:]]*//g`
                if [ `echo ${LINE} | awk '{ print $2 }'` = "\"sil\"" ]; then
            while [ `echo ${LINE} | awk '{ print $1 }'` != "<ENDHMM>" ];
            do
                echo ${LINE} >> ./interim_files/hmm4/hmmdefs.tmp1
                echo ${LINE} >> ./interim_files/hmm4/hmmdefs.tmp2
                                print_process ${NUM}
                                let "NUM += 1"
                                LINE=`cat -n ./interim_files/hmm4/hmmdefs | grep ^[[:space:]]*${NUM}[[:space:]] | sed s/^[[:space:]]*[[:digit:]]*//g`
                        done
                        
                        echo ${LINE} >> ./interim_files/hmm4/hmmdefs.tmp1

                        NUM2=1
                        while [ ${NUM2} != "28" ];
                        do
                                LINE2=`cat -n ./interim_files/hmm4/hmmdefs.tmp2 | grep ^[[:space:]]*${NUM2}[[:space:]] \
                                | sed s/^[[:space:]]*[[:digit:]]*//g`

                           case ${NUM2} in
                                        1 ) 
                                                echo ${LINE2} | sed s/~h\ \"sil\"/~h\ \"sp\"/g >> ./interim_files/hmm4/hmmdefs.tmp1
                                                ;;
                                        2 ) 
                                                echo ${LINE2} >> ./interim_files/hmm4/hmmdefs.tmp1
                                                ;;
                                        3 ) 
                                                echo ${LINE2} | sed s/5/3/g >> ./interim_files/hmm4/hmmdefs.tmp1
                                                ;;
                                        10 ) 
                                                echo ${LINE2} | sed s/3/2/g >> ./interim_files/hmm4/hmmdefs.tmp1
                                                ;;
                                        11 | 12 | 13 | 14 | 15)
                                                echo ${LINE2} >> ./interim_files/hmm4/hmmdefs.tmp1
                                                ;;
                                        22 )
                                                echo ${LINE2} | sed s/5/3/g >> ./interim_files/hmm4/hmmdefs.tmp1
                                                ;;
                                        24 ) 
                                                echo "0.000000e+000 1.000000e+000 0.000000e+000" >> ./interim_files/hmm4/hmmdefs.tmp1
                                                ;;
                                        25 ) 
                                                echo "0.000000e+000 0.900000e+000 0.100000e+000" >> ./interim_files/hmm4/hmmdefs.tmp1
                                                ;;
                                        26 ) 
                                                echo "0.000000e+000 0.000000e+000 0.000000e+000" >> ./interim_files/hmm4/hmmdefs.tmp1
                                                ;;
                                esac
                                let "NUM2 += 1"
                        done
        fi
                echo ${LINE} >> ./interim_files/hmm4/hmmdefs.tmp1
                print_process ${NUM}
                let "NUM += 1"
        done
        mv -f ./interim_files/hmm4/hmmdefs.tmp1 ./interim_files/hmm4/hmmdefs
        return 0
} 

make_dict1 () {
        cat ./interim_files/dict > ./interim_files/dict1
        echo "silence  []  sil" >> ./interim_files/dict1
        return 0
}

make_mktrihed () {
        if [ -f "$HTK/maketrihed" ]; then
                perl $HTK/maketrihed ./interim_files/monophones1 ./interim_files/triphones1
                mv mktri.hed ./interim_files/
        else
                echo "Error!! $HTK/maketrihed not found!"
                return 1
        fi
        return 0
}

make_treehed () {
        cat ./input_files/tree1.hed > ./interim_files/tree.hed
    perl $HTK/mkclscript.prl TB 350 ./interim_files/monophones0 >> ./interim_files/tree.hed
    echo ' ' >> ./interim_files/tree.hed             
    echo 'TR 1' >> ./interim_files/tree.hed
    echo ' ' >> ./interim_files/tree.hed
    echo 'AU "./interim_files/fulllist" ' >> ./interim_files/tree.hed
    echo 'CO "./interim_files/tiedlist" ' >> ./interim_files/tree.hed
    echo ' ' >> ./interim_files/tree.hed
    echo 'ST "./interim_files/trees" ' >> ./interim_files/tree.hed
        return 0
}

make_fulllist () {
        cat ./interim_files/fulllist > ./interim_files/fulllist-original
        cat ./interim_files/triphones1 >> ./interim_files/fulllist
        perl fixfulllist.pl ./interim_files/fulllist ./interim_files/fulllist1
        cat ./interim_files/fulllist1 > ./interim_files/fulllist
        rm ./interim_files/fulllist1
        return 0
}

print_heading () {
        if [ $? = "0" ]; then
                echo
                echo -e "\t$1"
                NUM=1
                while [ ${NUM} -lt 32 ];
                do
                        echo -n "=="
                        let "NUM += 1"
                done
                echo
        else
                exit 1
        fi
        return 0
}

########################################################################
#       Main 
########################################################################
print_heading "init"
        HTK=$1
        LexiconDirectory=$2     
        Debug=$3
        htk_init
        export PATH=$PATH:$HTK  # to access HTK executables
        
print_heading "Step 1 - Task Grammar"

        echo -e 'already completed manually\n'

print_heading "Step 2 - Pronunciation Dictionnary"
        make_wlist_label_file
    HDMan -A -D -T 1 -m -w ./interim_files/wlist -e ./input_files -n ./interim_files/monophones1 -i -l logs/Step2_HDMan1_log ./interim_files/dict $LexiconDirectory/VoxForge/VoxForgeDict > logs/Step2_HDMan2_log
    make_monophones0
        echo -e '***Please review the following HDMan output***:\n'
        cat logs/Step2_HDMan1_log
        
print_heading "Step 3 - Recording the Data"
        echo -e 'already completed manually\n'

print_heading "Step 4 - Creating Transcription Files"
        make_wordsmlf
#       HLEd -A -D -T 1 -l '*' -d ./interim_files/dict -i ./interim_files/phones0.mlf ./input_files/mkphones0.led ./interim_files/words.mlf > logs/Step4_HLEd_phones0_log
#       HLEd -A -D -T 1 -l '*' -d ./interim_files/dict -i ./interim_files/phones1.mlf ./input_files/mkphones1.led ./interim_files/words.mlf > logs/Step4_HLEd_phones1_log
        HLEd -A -D -T 1        -d ./interim_files/dict -i ./interim_files/phones0.mlf ./input_files/mkphones0.led ./interim_files/words.mlf > logs/Step4_HLEd_phones0_log
        HLEd -A -D -T 1        -d ./interim_files/dict -i ./interim_files/phones1.mlf ./input_files/mkphones1.led ./interim_files/words.mlf > logs/Step4_HLEd_phones1_log

print_heading "Step 5 - Coding the (Audio) Data"        
#       HCopy -A -D -T 1 -C ./input_files/wav_config -S master_codetrain_8kHz:16bit.scp > logs/Step5_HCopy_log
        echo -e 'already completed manually\n'
                
print_heading "Step 6 - Creating Monophones"
        perl ./prompts2trainscp.pl ./interim_files/master_prompts ./interim_files/train.scp

    echo -e 'making hmm0\n'
        HCompV -A -D -T 1 -C ./input_files/config -f 0.01 -m -S ./interim_files/train.scp -M ./interim_files/hmm0 input_files/proto > logs/Step6_HCompV_hmm0_log
        make_hmmdefs
        make_macros
        echo -e 'making hmm1\n'
        HERest -A -D -T 1 -C ./input_files/config -I ./interim_files/phones0.mlf -t 250.0 150.0 1000.0 -S ./interim_files/train.scp -H ./interim_files/hmm0/macros -H ./interim_files/hmm0/hmmdefs -M ./interim_files/hmm1 ./interim_files/monophones0 > logs/Step6_HERest_hmm1_log
        echo -e 'making hmm2\n'
        HERest -A -D -T 1 -C ./input_files/config -I ./interim_files/phones0.mlf -t 250.0 150.0 1000.0 -S ./interim_files/train.scp -H ./interim_files/hmm1/macros -H ./interim_files/hmm1/hmmdefs -M ./interim_files/hmm2 ./interim_files/monophones0 > logs/Step6_HERest_hmm2_log
        echo -e 'making hmm3\n'
        HERest -A -D -T 1 -C ./input_files/config -I ./interim_files/phones0.mlf -t 250.0 150.0 1000.0 -S ./interim_files/train.scp -H ./interim_files/hmm2/macros -H ./interim_files/hmm2/hmmdefs -M ./interim_files/hmm3 ./interim_files/monophones0 > logs/Step6_HERest_hmm3_log

print_heading "Step 7 - Fixing the Silence Model"
# using "cp ./interim_files/hmm3/. ./interim_files/hmm4 -R"
# causes these errors:
#cp: cannot create regular file `./interim_files/hmm4/./.svn/entries': Permission denied
#cp: cannot create regular file `./interim_files/hmm4/./.svn/text-base/macros.svn-base': Permission denied
#cp: cannot create regular file `./interim_files/hmm4/./.svn/text-base/hmmdefs.svn-base': Permission denied
cp ./interim_files/hmm3/hmmdefs ./interim_files/hmm4/hmmdefs
cp ./interim_files/hmm3/macros ./interim_files/hmm4/macros
# !!!!!!
        echo -e 'making hmm4\n'
        make_hmm4 2> /dev/null
        echo -e 'making hmm5\n'
        HHEd -A -D -T 1 -H ./interim_files/hmm4/macros -H ./interim_files/hmm4/hmmdefs -M ./interim_files/hmm5 ./input_files/sil.hed ./interim_files/monophones1 > logs/Step7_HHEd_hmm5_log
        echo -e 'making hmm6\n'
        HERest -A -D -T 1 -C ./input_files/config  -I ./interim_files/phones1.mlf -t 250.0 150.0 3000.0 -S ./interim_files/train.scp -H ./interim_files/hmm5/macros -H ./interim_files/hmm5/hmmdefs -M ./interim_files/hmm6 ./interim_files/monophones1 > logs/Step7_HERest_hmm6_log
        echo -e 'making hmm7\n'
        HERest -A -D -T 1 -C ./input_files/config  -I ./interim_files/phones1.mlf -t 250.0 150.0 3000.0 -S ./interim_files/train.scp -H ./interim_files/hmm6/macros -H ./interim_files/hmm6/hmmdefs -M ./interim_files/hmm7 ./interim_files/monophones1 > logs/Step7_HERest_hmm7_log

print_heading "Step 8 - Realigning the Training Data"
        make_dict1
        echo -e 'realign hmm7\n'
        HVite -A -D -T 1 -l '*' -o SWT -b silence -C ./input_files/config -H ./interim_files/hmm7/macros -H ./interim_files/hmm7/hmmdefs -i ./interim_files/aligned.mlf -m -t 250.0 150.0 1000.0 -y lab -a -I ./interim_files/words.mlf -S ./interim_files/train.scp ./interim_files/dict1 ./interim_files/monophones1 > logs/Step8_HVite_log
        echo -e '***Please review the HVite output log file: 'logs/Step8_HVite_log' ***:\n'
        echo -e 'if you find warning messages stating: "No tokens survived to final node of network at beam xxx.x"\n'
        echo -e 'listen to the wav audio file and ensure that it is properly transcribed in the corresponding label file\n'
#       cat logs/Step8_HVite_log
        echo -e 'making hmm8\n'
        HERest -A -D -T 1 -C ./input_files/config -I ./interim_files/aligned.mlf -t 250.0 150.0 3000.0 -S ./interim_files/train.scp -H ./interim_files/hmm7/macros -H ./interim_files/hmm7/hmmdefs -M ./interim_files/hmm8 ./interim_files/monophones1 > logs/Step8_HERest_hmm8_log
        echo -e 'making hmm9\n'
        HERest -A -D -T 1 -C ./input_files/config -I ./interim_files/aligned.mlf -t 250.0 150.0 3000.0 -S ./interim_files/train.scp -H ./interim_files/hmm8/macros -H ./interim_files/hmm8/hmmdefs -M ./interim_files/hmm9 ./interim_files/monophones1 > logs/Step8_HERest_hmm9_log

print_heading "Step 9 - Making Triphones from Monophones"
        echo -e 'making triphones\n'    
        HLEd -A -D -T 1 -n ./interim_files/triphones1 -l '*' -i ./interim_files/wintri.mlf ./input_files/mktri.led ./interim_files/aligned.mlf > logs/Step9_HLed_log
        make_mktrihed
        echo -e 'making hmm10\n'
        HHEd -A -D -T 1 -H ./interim_files/hmm9/macros -H ./interim_files/hmm9/hmmdefs -M ./interim_files/hmm10 ./interim_files/mktri.hed ./interim_files/monophones1 > logs/Step9_HHEd_hmm10_log
        echo -e 'making hmm11\n'
        HERest  -A -D -T 1 -C ./input_files/config -I ./interim_files/wintri.mlf -t 250.0 150.0 3000.0 -S ./interim_files/train.scp -H ./interim_files/hmm10/macros -H ./interim_files/hmm10/hmmdefs -M ./interim_files/hmm11 ./interim_files/triphones1 > logs/Step9_HERest_hmm11_log
        echo -e 'making hmm12\n'
        HERest  -A -D -T 1 -C ./input_files/config -I ./interim_files/wintri.mlf -t 250.0 150.0 3000.0 -s ./interim_files/stats -S ./interim_files/train.scp -H ./interim_files/hmm11/macros -H ./interim_files/hmm11/hmmdefs -M ./interim_files/hmm12 ./interim_files/triphones1 > logs/Step9_HERest_hmm12_log

print_heading "Step 10 - Making Tied-State Triphones"
        HDMan -A -D -T 1 -b sp -n ./interim_files/fulllist -g ./input_files/global.ded -l logs/Step10_HDMan_flog ./interim_files/dict-tri $LexiconDirectory/VoxForge/VoxForgeDict > logs/Step10_HDMan_log
        make_fulllist  
        make_treehed
        echo -e 'making hmm13\n'
        HHEd -A -D -T 1 -H ./interim_files/hmm12/macros -H ./interim_files/hmm12/hmmdefs -M ./interim_files/hmm13 ./interim_files/tree.hed ./interim_files/triphones1 > logs/Step10_HHed_hmm13_log
        echo -e 'making hmm14\n'
        HERest -A -D -T 1 -T 1 -C ./input_files/config -I ./interim_files/wintri.mlf -t 250.0 150.0 3000.0 -s ./interim_files/stats -S ./interim_files/train.scp -H ./interim_files/hmm13/macros -H ./interim_files/hmm13/hmmdefs -M ./interim_files/hmm14 ./interim_files/tiedlist > logs/Step10_HERest_hmm14_log
        echo -e 'making hmm15\n'
        HERest -A -D -T 1 -T 1 -C ./input_files/config -I ./interim_files/wintri.mlf -t 250.0 150.0 3000.0 -s ./interim_files/stats -S ./interim_files/train.scp -H ./interim_files/hmm14/macros -H ./interim_files/hmm14/hmmdefs -M ./interim_files/hmm15 ./interim_files/tiedlist > logs/Step10_HERest_hmm15_log

        cp ./interim_files/hmm15/hmmdefs ../acoustic_model_files
        cp ./interim_files/hmm15/macros ../acoustic_model_files 
        cp ./interim_files/tiedlist ../acoustic_model_files
        
print_heading "***completed***"