root/Trunk/Scripts/Testing_scripts/archive/Nightly_test.pm

#!/usr/bin/perl -w
####################################################################
###
### script name: Nightly_Test.pm
### modified by: Ken MacLean
### email: contact@voxforge.org
### Date: 2007.01.24
### Command: ./Nightly_Test.pm
### Version: 0.1
###             
### Copyright (C) 2007 Ken MacLean
###
### This program is free software; you can redistribute it and/or
### modify it under the terms of the GNU General Public License
### as published by the Free Software Foundation; either version 2
### of the License, or (at your option) any later version.
###
### This program is distributed in the hope that it will be useful,
### but WITHOUT ANY WARRANTY; without even the implied warranty of
### MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
### GNU General Public License for more details.
###
####################################################################
package Nightly_test;
use strict;
use File::Spec;
use POSIX qw(strftime);

my %parms;
$parms{"debug"} = 1;
$parms{"VoxForgeHome"} = "/home/kmaclean/workspace/VoxForge"; # for testing
#$parms{"VoxForgeHome"} = "/data/svn-mirror/Trunk";
        $parms{"AudioDirectory"} = $parms{"VoxForgeHome"}."/Audio";
        $parms{"LexiconDirectory"} = $parms{"VoxForgeHome"}."/Lexicon";
        $parms{"QSdir"} = $parms{"VoxForgeHome"}."/QuickStart";
        $parms{"ScriptsDirectory"} = $parms{"VoxForgeHome"}."/Scripts"; 
                $parms{"AudioScripts_dir"} = $parms{"ScriptsDirectory"}."/Audio_scripts";       
                        $parms{"Repository_dir"} = $parms{"AudioScripts_dir"}."/Repository";    
                        $parms{"UserSubmission_dir"} = $parms{"AudioScripts_dir"}."/UserSubmission";
                $parms{"HTK_dir"} = $parms{"ScriptsDirectory"}."/HTK/AMCreate_scripts"; 
                $parms{"MirroringScripts_dir"} = $parms{"ScriptsDirectory"}."/Mirroring_scripts";
                $parms{"Testing_scripts_dir"} = $parms{"ScriptsDirectory"}."/Testing_scripts";
my $year = strftime "%Y", localtime; #%Y      year (4 digits)
my $mon = strftime "%m", localtime; # %m      month number, starting with 01
my $mday = strftime "%d", localtime; #  %d      numeric day of the month, with leading zeros (eg 01..31)                        
#$parms{"mirror_dir"}  = "/data/svn-mirror"; 
$parms{"mirror_dir"}  = "/data/svn-mirror-copy"; # for testing
$parms{"Nightly_dir"}  = $parms{"mirror_dir"} . "/Nightly_Builds"; # for testing
        $parms{"Nightly_dirname"} = $parms{"Nightly_dir"}."/AcousticModel-$year-$mon-$mday";    
        $parms{"16kHz_16bit_NightlyAM"} = $parms{"Nightly_dirname"}."/HTK/16kHz_16bit/MFCC_O_D";                                
        $parms{"8kHz_16bit_NightlyAM"} = $parms{"Nightly_dirname"}."/HTK/8kHz_16bit/MFCC_O_D";          
$parms{"Tags"}  = $parms{"mirror_dir"} . "/Tags"; # for testing
        $parms{"Releases"} = $parms{"Tags"}."/Releases";        
        $parms{"CurrentBuild_dir"} = $parms{"Releases"}."/0_1_1-build726";
                        $parms{"16kHz_16bit_CurrentAM"} = $parms{"CurrentBuild_dir"}."/AcousticModels/HTK/16kHz-16bit/MFCC_O_D";                                
                        $parms{"8kHz_16bit_CurrentAM"} = $parms{"CurrentBuild_dir"}."/AcousticModels/HTK/8kHz-16bit/MFCC_O_D";                                  
$parms{"HTKBin"}  = "/usr/local/HTK"; # for testing
#$parms{"HTKBin"}  = "/usr/local/bin";
my $command;
####################################################################
my $debug = 0;
my $HTKbin = $parms{"HTKBin"};

my $Testing_scripts_dir = $parms{"Testing_scripts_dir"};
my $LexiconDirectory = $parms{"LexiconDirectory"} ;

# Step 1 - Acoustic Models
# path to AM: /data/svn-mirror/Nightly_Builds/AcousticModel-2007-01-24/HTK/16kHz_16bit/MFCC_O_D

# Step 2 - Create Test Prompts
$command = ("perl $HTKbin/prompts2mlf testref.mlf testprompts"); print "$command\n" if $debug; system($command);   

# Step 3 - Recording the Test Data
# path to audio: /home/kmaclean/workspace/VoxForge/Audio/Test/16kHz_16bit

# Step 4 - Coding the Data
# already done!
open(OUTPUT, ">TestResults") or die ("cannot write to TestResults file"); # open for output
print OUTPUT "Disclaimer: this is a *sanity test* only, and is *not* to be regarded as a valid test of the VoxForge Acoustic Models!
 * the audio files used for testing include only one voice, the Acoustic Models were also trained using that same voice, so the results will look better;
 * there are only 50 audio samples in the test database - not enough for a good test;
 * the VoxForge Acoustic Models are still alpha with respect to Speaker Independent Speech Recognition, so please donate some speech to VoxForge,
 thanks,
 Ken
 \n";
processNightly();
processCurrent();
close(OUTPUT);

# real time testing
#$command = ("julian -input mic -h $MFCC_files/hmmdefs -hlist $MFCC_files/tiedlist -smpFreq 16000 -C julian.jconf"); print "$command\n" if $debug; system($command) ;   
# no "_Z" (zero mean) permitted on HVite live? Live does not work! get "ERROR [+6320]  OpenAsChannel: cannot zero mean within buffer" error
#$command = ("HVite -A -D -T 1 -H $MFCC_files/macros -H $MFCC_files/hmmdefs -C HTK_Live_config -w wdnet -p 0.0 -s 5.0 $MFCC_files/dict $MFCC_files/tiedlist"); print "$command\n" if $debug; system($command) ;   

####################################################################
### Subroutines
####################################################################
sub processNightly {
        my $Nightly_dirname = $parms{"Nightly_dirname"};
        print OUTPUT "Testing Acoustic Models created in: $Nightly_dirname\n\n";
        HTK_16kHz_16bit($parms{"16kHz_16bit_NightlyAM"});
        Julian_16kHz_16bit($parms{"16kHz_16bit_NightlyAM"});
        HTK_8kHz_16bit($parms{"8kHz_16bit_NightlyAM"});
        Julian_8kHz_16bit($parms{"8kHz_16bit_NightlyAM"});
        PrintNotes();
}

sub processCurrent {
        my $CurrentBuild_dir = $parms{"CurrentBuild_dir"};
        print OUTPUT "\n\n\n================================================================================================================\n";
        print OUTPUT "For comparison purposes, see below for the same Tests on the most current release of the VoxForge Acoustic Models:
($CurrentBuild_dir)\n";
        print OUTPUT "================================================================================================================\n";
        HTK_16kHz_16bit($parms{"16kHz_16bit_CurrentAM"});
        Julian_16kHz_16bit($parms{"16kHz_16bit_CurrentAM"});
        HTK_8kHz_16bit($parms{"8kHz_16bit_CurrentAM"});
        Julian_8kHz_16bit($parms{"8kHz_16bit_CurrentAM"});
        PrintNotes();
}

sub HTK_16kHz_16bit {
        my ($AM_Dir) = @_;      
        my $wordinspen = "0.0";
        my $gramscfact = "1.0";
        system("rm -f recout.mlf"); 
        $command = ("$HTKbin/HVite -A -D -T 1 -H $AM_Dir/macros -H $AM_Dir/hmmdefs -C $Testing_scripts_dir/wav_config -S $Testing_scripts_dir/wavlst16kHz_16bit -l '*' -i recout.mlf -w $Testing_scripts_dir/wdnet -p $wordinspen -s $gramscfact $LexiconDirectory/VoxForge/VoxForgeDict $AM_Dir/tiedlist  1>/dev/null 2>/dev/null");  system($command);   
        my $HTK_TestResults = `$HTKbin/HResults -I testref.mlf $AM_Dir/tiedlist recout.mlf`;
        $HTK_TestResults =~ s/ HTK//;
        print OUTPUT ("HTK 16kHz_16bit\n");    
        print OUTPUT ("---------------\n");     
        print OUTPUT ("\tword insertion penalty: $wordinspen\n"); 
        print OUTPUT ("\tgrammar scale factor: $gramscfact\n");         
        print OUTPUT "$HTK_TestResults\n";
}

sub Julian_16kHz_16bit {        
        my ($AM_Dir) = @_;      
        my $penalty1="0.5";
        my $penalty2="100.0";
        my $iwsppenalty = "-55.0";
        system("rm -f julianProcessed");                
        $command = ("julian -penalty1 $penalty1 -penalty2 $penalty2 -iwsppenalty $iwsppenalty -input rawfile -filelist wavlst16kHz_16bit -h $AM_Dir/hmmdefs -hlist $AM_Dir/tiedlist -smpFreq 16000  -C julian.jconf  > julianOutput 2>/dev/null"); print "$command\n" if $debug; system($command) ;   
        $command = ("perl ./ProcessJulianOutput.pl julianOutput julianProcessed"); print "$command\n" if $debug; system($command) ;   
        my $Julian_TestResults = `$HTKbin/HResults -I testref.mlf $AM_Dir/tiedlist julianProcessed` ; 
        $Julian_TestResults =~ s/ HTK//;
        print OUTPUT ("Julian 16kHz_16bit\n"); 
        print OUTPUT ("------------------\n");           
        print OUTPUT ("\tword insertion penalty\n");  
        print OUTPUT ("\t  first pass (-penalty1):$penalty1\n");  
        print OUTPUT ("\t  second pass (-penalty2):$penalty2\n");  
        print OUTPUT ("\ttransition penalty:$iwsppenalty (for short-term inter-word pauses between words (-iwsppenalty))\n");           
        print OUTPUT "$Julian_TestResults\n";
}

sub HTK_8kHz_16bit {
        my ($AM_Dir) = @_;              
        my $wordinspen = "10.0";
        my $gramscfact = "5.0";
        system("rm -f recout.mlf");             
        $command = ("$HTKbin/HVite -A -D -T 1 -H $AM_Dir/macros -H $AM_Dir/hmmdefs -C $Testing_scripts_dir/wav_config -S $Testing_scripts_dir/wavlst8kHz_16bit -l '*' -i recout.mlf -w $Testing_scripts_dir/wdnet -p $wordinspen -s $gramscfact $LexiconDirectory/VoxForge/VoxForgeDict $AM_Dir/tiedlist 1>/dev/null 2>/dev/null");  system($command);   
        my $HTK_TestResults = `$HTKbin/HResults -I testref.mlf $AM_Dir/tiedlist recout.mlf`;
        $HTK_TestResults =~ s/ HTK//;
        print OUTPUT ("HTK 8kHz_16bit\n");  
        print OUTPUT ("---------------\n");     
        print OUTPUT ("\tword insertion penalty: $wordinspen\n"); 
        print OUTPUT ("\tgrammar scale factor: $gramscfact\n");         
        print OUTPUT "$HTK_TestResults\n";
}

sub Julian_8kHz_16bit {
        my ($AM_Dir) = @_;                      
        my $penalty1="50.0";
        my $penalty2="100.0";
        my $iwsppenalty = "-55.0";
        system("rm -f julianProcessed");                
        $command = ("julian -penalty1 $penalty1 -penalty2 $penalty2 -iwsppenalty $iwsppenalty -input rawfile -filelist wavlst8kHz_16bit -h $AM_Dir/hmmdefs -hlist $AM_Dir/tiedlist -smpFreq 8000  -C julian.jconf  > julianOutput 2>/dev/null"); print "$command\n" if $debug; system($command) ;   
        $command = ("perl ./ProcessJulianOutput.pl julianOutput julianProcessed"); print "$command\n" if $debug; system($command) ;   
        my $Julian_TestResults = `$HTKbin/HResults -I testref.mlf $AM_Dir/tiedlist julianProcessed` ; 
        $Julian_TestResults =~ s/ HTK//;
        print OUTPUT ("Julian 8kHz_16bit\n");  
        print OUTPUT ("------------------\n");          
        print OUTPUT ("\tword insertion penalty\n");  
        print OUTPUT ("\t  first pass (-penalty1):$penalty1\n");  
        print OUTPUT ("\t  second pass (-penalty2):$penalty2\n");  
        print OUTPUT ("\ttransition penalty::$iwsppenalty (for short-term inter-word pauses between words (-iwsppenalty))\n");                  
        print OUTPUT "$Julian_TestResults\n";
}       

sub PrintNotes {
print OUTPUT "Notes:

    * the line starting with SENT gives the percentage of sentences that were recognized correctly, out of N sentences in total.
    * the line starting with WORD gives the percentage of words that were recognized correctly, out of N words in total
      However, since HTK or Julius erroneously 'added' words that are not in the audio file (i.e. insertion errors) they usually get a lower percentage accuracy rating.
    * Count definitions:
          o D - Deletion Error
          o S - Substitution Error
          o I - Insertion Error\n";
}

1;