voxforge.org
VoxForge Dev
Home · Read · Listen · Forums · Dev · Downloads · About

root/Trunk/Scripts/Testing_scripts/NightlyTest.pm

Revision 2484, 12.1 kB (checked in by kmaclean, 9 months ago)

audio processing for Russian, German and Dutch - snapshot

Line 
1 #!/usr/bin/perl -w
2 ####################################################################
3 ###
4 ### script name: NightlyTest.pm
5 ### modified by: Ken MacLean
6 ### email: contact@voxforge.org
7 ### Date: 2007.01.24
8 ### Version: 0.1
9 ###             
10 ### Copyright (C) 2007 Ken MacLean
11 ###
12 ### This program is free software; you can redistribute it and/or
13 ### modify it under the terms of the GNU General Public License
14 ### as published by the Free Software Foundation; either version 3
15 ### of the License, or (at your option) any later version.
16 ###
17 ### This program is distributed in the hope that it will be useful,
18 ### but WITHOUT ANY WARRANTY; without even the implied warranty of
19 ### MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
20 ### GNU General Public License for more details.
21 ###
22 ####################################################################
23 package NightlyTest;
24 use strict;
25 use File::Spec;
26 use POSIX qw(strftime);
27
28 my $command;
29 ####################################################################
30 sub Process {
31         my ($parms ) = @_;
32         # !!!!!!
33         $parms->setSpeechCorpusLanguage('EN');
34         # !!!!!!       
35                 my $debug = $$parms{"debug"};   
36                 my $HTKbin = $$parms{"HTKBin"};
37                 my $Nightly_test_dir = $$parms{"Nightly_test_dir"};     
38         print ("Process CurrentAM_16kHz_16bit:" . $$parms{"CurrentAM_16kHz_16bit"} . ":\n") if $debug;         
39         $command = ("cd $Nightly_test_dir && perl $HTKbin/prompts2mlf testref.mlf testprompts"); print "$command\n" if $debug; system($command);   
40 my $disclaimer = "DISCLAIMER: this is a *sanity test* only, and is *not* to be regarded as a valid test of the VoxForge Acoustic Models!
41  * the speech files used for testing use a single person's voice;
42  * the same person also contributed speech for the creation of the VoxForge the Acoustic Models;
43  * there are only 50 audio samples in the test database - not enough for a good test;
44  * the VoxForge Acoustic Models are still an alpha release with respect to Speaker Independent Speech Recognition, so please donate some speech to VoxForge,
45
46  thanks,
47
48  Ken
49 \n";
50
51         process16kHz_16bit($parms, $disclaimer) || die "NightlyTest error: $?";
52         process8kHz_16bit($parms, $disclaimer)|| die "NightlyTest error: $?";
53         return 1;   
54 }
55 ####################################################################
56 ### Subroutines
57 ####################################################################
58 sub process16kHz_16bit {
59         my ($parms, $disclaimer ) = @_;
60                 my $debug = $$parms{"debug"};
61                 my $Nightly_dirname = $$parms{"Nightly_dirname"};
62                 my $NightlyAM_16kHz_16bit = $$parms{"NightlyAM_16kHz_16bit"};
63                 my $CurrentBuild_dir = $$parms{"CurrentBuild_dir"};     
64                 my $CurrentAM_16kHz_16bit = $$parms{"CurrentAM_16kHz_16bit"};
65         print ("Process CurrentAM_16kHz_16bit:" . $$parms{"CurrentAM_16kHz_16bit"} . ":\n") if $debug;
66         open(OUTPUT, ">$NightlyAM_16kHz_16bit/TestResults") or die ("cannot write to TestResults file"); # open for output                     
67         print OUTPUT "$disclaimer\n";   
68         print OUTPUT "Testing Acoustic Models created in: $Nightly_dirname\n\n";
69         HTK_16kHz_16bit($parms, $NightlyAM_16kHz_16bit) || die "NightlyTest error: $?";
70         Julian_16kHz_16bit($parms, $NightlyAM_16kHz_16bit) || die "NightlyTest error: $?";
71         PrintNotes();   
72         print OUTPUT "\n\n\n================================================================================================================\n";
73         print OUTPUT "For comparison purposes, see below for the same Tests on the most current release of the VoxForge Acoustic Models:
74 ($CurrentBuild_dir)\n";
75         print OUTPUT "================================================================================================================\n";
76         HTK_16kHz_16bit($parms, $CurrentAM_16kHz_16bit) || die "NightlyTest error: $?";
77         Julian_16kHz_16bit($parms, $CurrentAM_16kHz_16bit) || die "NightlyTest error: $?";
78         PrintNotes();
79         close(OUTPUT);
80         return 1;   
81 }
82
83 sub process8kHz_16bit {
84         my ($parms, $disclaimer ) = @_;
85                 my $debug = $$parms{"debug"};
86                 my $Nightly_dirname = $$parms{"Nightly_dirname"};
87                 my $NightlyAM_8kHz_16bit = $$parms{"NightlyAM_8kHz_16bit"};
88                 my $CurrentBuild_dir = $$parms{"CurrentBuild_dir"};                     
89                 my $CurrentAM_8kHz_16bit = $$parms{"CurrentAM_8kHz_16bit"};             
90         open(OUTPUT, ">$NightlyAM_8kHz_16bit/TestResults") or die ("cannot write to TestResults file"); # open for output                       
91         print OUTPUT "$disclaimer\n";   
92         HTK_8kHz_16bit($parms, $NightlyAM_8kHz_16bit) || die "NightlyTest error: $?";
93         Julian_8kHz_16bit($parms, $NightlyAM_8kHz_16bit) || die "NightlyTest error: $?";
94         PrintNotes();   
95         print OUTPUT "\n\n\n================================================================================================================\n";
96         print OUTPUT "For comparison purposes, see below for the same Tests on the most current release of the VoxForge Acoustic Models:
97 ($CurrentBuild_dir)\n";
98         print OUTPUT "================================================================================================================\n";
99         print OUTPUT "$disclaimer\n";
100         HTK_8kHz_16bit($parms, $CurrentAM_8kHz_16bit) || die "NightlyTest error: $?";
101         Julian_8kHz_16bit($parms, $CurrentAM_8kHz_16bit) || die "NightlyTest error: $?";
102         PrintNotes();
103         close(OUTPUT); 
104         return 1;               
105 }
106
107 sub HTK_16kHz_16bit {
108         my ($parms, $AM_Dir ) = @_;
109                 my $debug = $$parms{"debug"};
110                 my $LexiconDirectory = $$parms{"LexiconDirectory"};
111                 my $HTKbin = $$parms{"HTKBin"} ;
112                 my $Nightly_test_dir = $$parms{"Nightly_test_dir"};     
113                 my $testing = $$parms{"testing"};               
114         my $wordinspen = "0.0";
115         my $gramscfact = "1.0";
116         system("rm -f recout.mlf");
117         if ($testing) {$command = ("cd $Nightly_test_dir && $HTKbin/HVite -A -D -T 1 -H $AM_Dir/macros -H $AM_Dir/hmmdefs -C $Nightly_test_dir/wav_config -S $Nightly_test_dir/test_wavlst16kHz_16bit -l '*' -i recout.mlf -w $Nightly_test_dir/wdnet -p $wordinspen -s $gramscfact $LexiconDirectory/VoxForge/VoxForgeDict $AM_Dir/tiedlist  1>/dev/null 2>/dev/null");  system($command);  }
118                          else {$command = ("cd $Nightly_test_dir && $HTKbin/HVite -A -D -T 1 -H $AM_Dir/macros -H $AM_Dir/hmmdefs -C $Nightly_test_dir/wav_config -S $Nightly_test_dir/wavlst16kHz_16bit -l '*' -i recout.mlf -w $Nightly_test_dir/wdnet -p $wordinspen -s $gramscfact $LexiconDirectory/VoxForge/VoxForgeDict $AM_Dir/tiedlist  1>/dev/null 2>/dev/null");  system($command);  }
119         my $HTK_TestResults = `cd $Nightly_test_dir &&  $HTKbin/HResults -I testref.mlf $AM_Dir/tiedlist recout.mlf`;
120         $HTK_TestResults =~ s/ HTK//;
121         print OUTPUT ("HTK 16kHz_16bit\n");   
122         print OUTPUT ("---------------\n");   
123         print OUTPUT ("  Parameters:\n");               
124         print OUTPUT ("\tword insertion penalty: $wordinspen\n");
125         print OUTPUT ("\tgrammar scale factor: $gramscfact\n");         
126         print OUTPUT "$HTK_TestResults\n";
127         return 1;       
128 }
129
130 sub Julian_16kHz_16bit {       
131         my ($parms, $AM_Dir ) = @_;
132                 my $debug = $$parms{"debug"};           
133                 my $HTKbin = $$parms{"HTKBin"} ;
134                 my $JulianBin = $$parms{"JulianBin"};
135                 my $Nightly_test_dir = $$parms{"Nightly_test_dir"};     
136                 my $testing = $$parms{"testing"};                       
137         my $penalty1="0.5";
138         my $penalty2="100.0";
139         my $iwsppenalty = "-55.0";
140         system("rm -f julianProcessed");               
141         if ($testing) {$command = ("cd $Nightly_test_dir &&  $JulianBin/julian -penalty1 $penalty1 -penalty2 $penalty2 -iwsppenalty $iwsppenalty -input rawfile -filelist test_wavlst16kHz_16bit -h $AM_Dir/hmmdefs -hlist $AM_Dir/tiedlist -smpFreq 16000  -C julian.jconf  > julianOutput 2>/dev/null"); print "$command\n" if $debug; system($command) ;  }
142                          else {$command = ("cd $Nightly_test_dir &&  $JulianBin/julian -penalty1 $penalty1 -penalty2 $penalty2 -iwsppenalty $iwsppenalty -input rawfile -filelist wavlst16kHz_16bit -h $AM_Dir/hmmdefs -hlist $AM_Dir/tiedlist -smpFreq 16000  -C julian.jconf  > julianOutput 2>/dev/null"); print "$command\n" if $debug; system($command) ;  }
143         $command = ("cd $Nightly_test_dir &&  perl ./ProcessJulianOutput.pl julianOutput julianProcessed"); print "$command\n" if $debug; system($command) ;   
144         my $Julian_TestResults = `cd $Nightly_test_dir &&  $HTKbin/HResults -I testref.mlf $AM_Dir/tiedlist julianProcessed` ;
145         $Julian_TestResults =~ s/ HTK//;
146         print OUTPUT ("Julian 16kHz_16bit\n");
147         print OUTPUT ("------------------\n");           
148         print OUTPUT ("  Parameters:\n");       
149         print OUTPUT ("\tword insertion penalty\n"); 
150         print OUTPUT ("\t  first pass (-penalty1):$penalty1\n"); 
151         print OUTPUT ("\t  second pass (-penalty2):$penalty2\n"); 
152         print OUTPUT ("\ttransition penalty (-iwsppenalty):$iwsppenalty (for short-term inter-word pauses between words)\n");           
153         print OUTPUT "$Julian_TestResults\n";
154         return 1;       
155 }
156
157 sub HTK_8kHz_16bit {
158         my ($parms, $AM_Dir ) = @_;
159                 my $debug = $$parms{"debug"};           
160                 my $LexiconDirectory = $$parms{"LexiconDirectory"};
161                 my $HTKbin = $$parms{"HTKBin"} ;
162                 my $Nightly_test_dir = $$parms{"Nightly_test_dir"};     
163                 my $testing = $$parms{"testing"};               
164         my $wordinspen = "10.0";
165         my $gramscfact = "5.0";
166         system("rm -f recout.mlf");     
167         if ($testing) {$command = ("cd $Nightly_test_dir &&  $HTKbin/HVite -A -D -T 1 -H $AM_Dir/macros -H $AM_Dir/hmmdefs -C $Nightly_test_dir/wav_config -S $Nightly_test_dir/test_wavlst8kHz_16bit -l '*' -i recout.mlf -w $Nightly_test_dir/wdnet -p $wordinspen -s $gramscfact $LexiconDirectory/VoxForge/VoxForgeDict $AM_Dir/tiedlist 1>/dev/null 2>/dev/null");  system($command);  }
168                          else {$command = ("cd $Nightly_test_dir &&  $HTKbin/HVite -A -D -T 1 -H $AM_Dir/macros -H $AM_Dir/hmmdefs -C $Nightly_test_dir/wav_config -S $Nightly_test_dir/wavlst8kHz_16bit -l '*' -i recout.mlf -w $Nightly_test_dir/wdnet -p $wordinspen -s $gramscfact $LexiconDirectory/VoxForge/VoxForgeDict $AM_Dir/tiedlist 1>/dev/null 2>/dev/null");  system($command);           }
169         my $HTK_TestResults = `cd $Nightly_test_dir &&  $HTKbin/HResults -I testref.mlf $AM_Dir/tiedlist recout.mlf`;
170         $HTK_TestResults =~ s/ HTK//;
171         print OUTPUT ("HTK 8kHz_16bit\n"); 
172         print OUTPUT ("---------------\n");     
173         print OUTPUT ("  Parameters:\n");       
174         print OUTPUT ("\tword insertion penalty: $wordinspen\n");
175         print OUTPUT ("\tgrammar scale factor: $gramscfact\n");         
176         print OUTPUT "$HTK_TestResults\n";
177         return 1;   
178 }
179
180 sub Julian_8kHz_16bit {
181         my ($parms, $AM_Dir ) = @_;
182                 my $debug = $$parms{"debug"};           
183                 my $HTKbin = $$parms{"HTKBin"} ;
184                 my $JulianBin = $$parms{"JulianBin"};           
185                 my $Nightly_test_dir = $$parms{"Nightly_test_dir"};
186                 my $testing = $$parms{"testing"};                                               
187         my $penalty1="50.0";
188         my $penalty2="100.0";
189         my $iwsppenalty = "-55.0";
190         system("rm -f julianProcessed");               
191         if ($testing) {$command = ("cd $Nightly_test_dir &&  $JulianBin/julian -penalty1 $penalty1 -penalty2 $penalty2 -iwsppenalty $iwsppenalty -input rawfile -filelist test_wavlst8kHz_16bit -h $AM_Dir/hmmdefs -hlist $AM_Dir/tiedlist -smpFreq 8000  -C julian.jconf  > julianOutput 2>/dev/null"); print "$command\n" if $debug; system($command) ;   }
192                          else {$command = ("cd $Nightly_test_dir &&  $JulianBin/julian -penalty1 $penalty1 -penalty2 $penalty2 -iwsppenalty $iwsppenalty -input rawfile -filelist wavlst8kHz_16bit -h $AM_Dir/hmmdefs -hlist $AM_Dir/tiedlist -smpFreq 8000  -C julian.jconf  > julianOutput 2>/dev/null"); print "$command\n" if $debug; system($command) ;   }
193         $command = ("cd $Nightly_test_dir &&  perl ./ProcessJulianOutput.pl julianOutput julianProcessed"); print "$command\n" if $debug; system($command) ;   
194         my $Julian_TestResults = `cd $Nightly_test_dir &&  $HTKbin/HResults -I testref.mlf $AM_Dir/tiedlist julianProcessed` ;
195         $Julian_TestResults =~ s/ HTK//;
196         print OUTPUT ("Julian 8kHz_16bit\n"); 
197         print OUTPUT ("------------------\n");         
198         print OUTPUT ("  Parameters:\n");       
199         print OUTPUT ("\tword insertion penalty\n"); 
200         print OUTPUT ("\t  first pass (-penalty1):$penalty1\n"); 
201         print OUTPUT ("\t  second pass (-penalty2):$penalty2\n"); 
202         print OUTPUT ("\ttransition penalty (-iwsppenalty):$iwsppenalty (for short-term inter-word pauses between words)\n");           
203         print OUTPUT "$Julian_TestResults\n";
204         return 1;   
205 }       
206
207 sub PrintNotes {
208 print OUTPUT "Notes:
209
210     * the line starting with SENT gives the percentage of sentences that were recognized correctly, out of N sentences in total.
211     * the line starting with WORD gives the percentage of words that were recognized correctly, out of N words in total
212       However, since HTK or Julius erroneously 'added' words that are not in the audio file (i.e. insertion errors) they usually get a lower percentage accuracy rating.
213     * Count definitions:
214           o D - Deletion Error
215           o S - Substitution Error
216           o I - Insertion Error\n";
217         return 1;     
218 }
219 1;
Note: See TracBrowser for help on using the browser.
©2005-2008 VoxForge