Changeset 2586

Timestamp:

05/15/08 10:44:44 (5 months ago)

Author:

kmaclean

Message:

AudioSegmentation script - alpha release

Files:

• Trunk/Scripts/Audio_scripts/AudioSegmentation/AudioBook.pm (modified) (9 diffs)
• Trunk/Scripts/Audio_scripts/AudioSegmentation/AudioBook/Audio.pm (modified) (8 diffs)
• Trunk/Scripts/Audio_scripts/AudioSegmentation/AudioBook/Dictionary.pm (modified) (5 diffs)
• Trunk/Scripts/Audio_scripts/AudioSegmentation/AudioBook/Text.pm (modified) (14 diffs)
• Trunk/Scripts/Audio_scripts/AudioSegmentation/AudioBook/interim_files/dlog1 (deleted)
• Trunk/Scripts/Audio_scripts/AudioSegmentation/AudioBook/interim_files/dlog2 (deleted)
• Trunk/Scripts/Audio_scripts/AudioSegmentation/AudioBook/interim_files/logs (added)
• Trunk/Scripts/Audio_scripts/AudioSegmentation/AudioBook/interim_files/logs/HVite_log (added)
• Trunk/Scripts/Audio_scripts/AudioSegmentation/AudioBook/interim_files/logs/dlog1 (added)
• Trunk/Scripts/Audio_scripts/AudioSegmentation/AudioBook/interim_files/logs/dlog2 (added)
• Trunk/Scripts/Audio_scripts/AudioSegmentation/AudioBook/output_files/AudioBook_Log (added)
• Trunk/Scripts/Audio_scripts/AudioSegmentation/AudioBook/output_files/prompts (modified) (1 diff)

Trunk/Scripts/Audio_scripts/AudioSegmentation/AudioBook.pm

@@ -36 +36 @@
-### Class Variables
-####################################################################
-r,$opt_t,$opt_x,$opt_K
+m,$opt_p,$opt_r,$opt_s,$opt_t,$opt_x,$opt_S
-my %self;
-$self{'debug'} = 0;
-$self{'g2p_model'} = "AudioBook/input_files/g2p/models/model-5";
-$self{'htk_files'} = "AudioBook/input_files/htk";
+$self{'log'} = "AudioBook/output_files/AudioBook_Log";
+my $default_average_sentence_length = 13;
+my $default_max_sentence_length = 18;
+my $default_min_pause_for_sentence_break = 2000000;
-my $command;
-
@@ -46 +50 @@
-### Main
-####################################################################
+cleanupFiles(\%self);
-getOptions(\%self);
-process(\%self);
@@ -62 +67 @@
-        my $voxforgeDict = $self->{"voxforgeDict"};
-        my $htk_files = $self->{'htk_files'};
-
-
+
+
+
-        $textContents->createWLISTFile("AudioBook/interim_files/wlist");
-        
@@ -73 +79 @@
-                # need to update dict with missing words
-                # can't seem to change default HDMan log file with "-l" parameter
-                $command = ("cp AudioBook/interim_files/dlog AudioBook/interim_files/dlog1"); print "cmd:$command\n" if $debug; system($command);
-                $command = ("HDMan -A -D -T 1 -g $htk_files/global.ded -m -w AudioBook/interim_files/wlist -i -l AudioBook/interim_files/dlog AudioBook/interim_files/dict $voxforgeDict"); system($command) == 0 or confess "fullrun $command failed: $?";
-
-
+logs/
+# no longer required
-        } else {
-
-
-
-
+
+
+
+
+
-        } 
-        $command = ("cp AudioBook/interim_files/dict AudioBook/output_files"); print "cmd:$command\n" if $debug; system($command);      
@@ -88 +94 @@
-        if (defined($tarSuffix)){
-                _createTarFile($self);
+        }
+}
+
+sub cleanupFiles {
+        my ($self)= @_;
+        if (defined(<AudioBook/interim_files/*>)) {
+                unlink (<AudioBook/interim_files/*>);
+        }
+        if (defined(<AudioBook/output_files/wav/*>)) {
+                unlink (<AudioBook/output_files/wav/*>);        
+        }
+        if (defined(<AudioBook/output_files/logs/*>)) {
+                unlink (<AudioBook/output_files/logs/*>);       
+        }
+        if (defined(<AudioBook/output_files/*>)) {      
+                unlink (<AudioBook/output_files/*>);
-        }
-}
@@ -122 +144 @@
-sub getOptions {
-        my ($self)= @_;
-
+
+
-        if ($opt_a and $opt_t) {        
-                if (-r $opt_a) {
@@ -143 +166 @@
-                        $self->{"voxforgeDict"}="AudioBook/input_files/VoxForgeDict";   
-                }
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
-                if (defined($opt_T)) {
-                        if ($opt_x) {
@@ -174 +213 @@
-                        }
-                }
-K) {
+S) { # Sanity test switch
-                $self->{"audiofile"}="AudioBook/test/audio.wav";
-                #$self->{"textfile"}="AudioBook/test/text-simple.txt";
-                $self->{"textfile"}="AudioBook/test/text-original.txt";
-
+print "cmd:$command\n"; 
-                $self->{"voxforgeDict"}="AudioBook/interim_files/VoxForgeDict";
-                $self->{"tarSuffix"}=_random_characters(3);
-                $self->{"username"}="test";
+                $self->{"average_sentence_length"}= $default_average_sentence_length;
+                $self->{"max_sentence_length"}= $default_max_sentence_length;
+                $self->{"min_pause_for_sentence_break"}=$default_min_pause_for_sentence_break;
-        } elsif ($opt_h) {
-                print "\nVoxForge Audio Segmentation Script Parameters\n";      
-                print   "=============================================\n";      
-
- 
+ (WAV format only)
+
-                print "-h\tshow help\n";        
-
+
+
+
-                print "-r\tREADME file (default = AudioBook/input_files/README)\n";             
+                print "-s\tAverage sentence length (default = $default_average_sentence_length words)\n";                               
-                print "-t\t* text file name\n";
-                print "-u\tusername or name you want file stats collected by on VoxForge Metrics \n";
-                print "\tpage:\t(http://www.voxforge.org/home/downloads/metrics)\n";    
-                print "-x\tunique tar file suffix (max 3 characters - remainder is truncated)\n";
-K\trun
+S\trun sanity
-                print "-T\tcreate gzipped/tar file\n";
-                print "\n\t* required for script to run\n";     
@@ -210 +255 @@
-}
-
+####################################################################
+### Gettors - Public
+####################################################################
+sub getAverage_sentence_length {
+        my $self = shift;
+        return $self->{"average_sentence_length"};
+}
+
+sub getMax_sentence_length {
+        my $self = shift;
+        return $self->{"max_sentence_length"};
+}
+
+sub getMin_pause_for_sentence_break {
+        my $self = shift;
+        return $self->{"max_sentence_length"};
+}
+
-1;

Trunk/Scripts/Audio_scripts/AudioSegmentation/AudioBook/Audio.pm

@@ -38 +38 @@
-### Class Variables
-####################################################################
-
-
-
-
+
+
+
-my (@max_sentences, $max_sentence_length_found, $max_sentence_length_linenumber, $min_sentence_length_linenumber);
-
+
+
-my $up_increment = 1;
-my $down_increment = -1;
-($command)
+$command
-####################################################################
-### Constructor
@@ -57 +57 @@
-        $self{'htk_files'} = $super->{'htk_files'};
-        $self{'g2p_model'} = $super->{'g2p_model'};
-
+
+
+
+
+
-        bless(\%self,$class);
-        return \%self;
@@ -87 +91 @@
-        $textContents->createMLFFile("downsampled","AudioBook/interim_files/words.mlf" );
-        # forced alignment - creates aligned.out
-
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
-}
-
@@ -135 +156 @@
-        my ($self,$filename,$textContents) = @_;        
-        my $debug = $self->{"debug"};           
-
+
+
-        my $filename_nopath = basename($filename);
-        my $filename_nosuffix = fileparse($filename, "wav");
@@ -144 +166 @@
-        $self->{"filename_prefix"} = lc(substr($filename_nopath,0,3));
-        $self->{"textContents"} = $textContents;
+        
+        my $average_sentence_length = $self->{"average_sentence_length"};
+        my $max_sentence_length = $self->{"max_sentence_length"};
+        my $min_pause_for_sentence_break = $self->{"min_pause_for_sentence_break"};
-
-        my $samplerate = _getSampleRate($self);
@@ -200 +226 @@
-        
-        $command = ("rm AudioBook/interim_files/temp.wav"); print "cmd:$command\n" if $debug; system($command);
-
-
-
-
-
-
-
-
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
-}
-
@@ -216 +249 @@
-        my $debug = $self->{"debug"};   
-        my $filename_prefix = $self->{"filename_prefix"};
+        my $average_sentence_length = $self->{"average_sentence_length"};
+        #my $max_sentence_length = $self->{"max_sentence_length"};
+        my $min_pause_for_sentence_break = $self->{"min_pause_for_sentence_break"};
-        
-        my @aligned_line = split(/\s/,$$aligned_words[$$sentence_end + $increment]);
@@ -277 +313 @@
-        my ($self,$sentence_start,$sentence_end,$increment,$filename_prefix,$padded_fileid ) = @_;
-        my $debug = $self->{"debug"};   
+        #my $average_sentence_length = $self->{"average_sentence_length"};
+        my $max_sentence_length = $self->{"max_sentence_length"};
+        #my $min_pause_for_sentence_break = $self->{"min_pause_for_sentence_break"};
-        
-        if ($debug) {

Trunk/Scripts/Audio_scripts/AudioSegmentation/AudioBook/Dictionary.pm

@@ -38 +38 @@
-        $self{'htk_files'} = $super->{'htk_files'};
-        $self{'g2p_model'} = $super->{'g2p_model'};
+        $self{'log'} = $super->{'log'}; 
-        bless(\%self,$class);
-        return \%self;
@@ -52 +53 @@
-        $self->{"missing_words"} = $missing_words;
-         
-
-
+
+
+
+
-        open(MISSINGWORDS,">$missing_words") or confess ("cannot open $missing_words file");
-        my $missingwordsheader = 0;
@@ -82 +85 @@
-sub getPronunciations { # public 
-        my ($self,$missing_words_out)= @_;
-"debug"
+'debug'
-        my $model = $self->{'g2p_model'};
-
+
+
-
-        $self->{"missing_words_out"} = $missing_words_out;
@@ -94 +98 @@
-        }
-        open(MISSINGWORDSOUT,">$missing_words_out") or confess ("cannot open $missing_words_out file");
+        open(LOG,">>$log") or confess ("cannot open $log file");
+        print LOG "\nMissing Words added to Pronunciation Dictionary:\n";       
+        print LOG "------------------------------------------------\n";         
-        my ($word, $phonemes);
-        format MISSINGWORDSOUT =
+@<<<<<<<<<<<<<<<@<<<<<<<<<<<<<<<@<<<<<<<<<<<<<<<<<<<<<<<<<<<<<
+$word,"[" . $word ."]",$phonemes
+.
+        format LOG =
-@<<<<<<<<<<<<<<<@<<<<<<<<<<<<<<<@<<<<<<<<<<<<<<<<<<<<<<<<<<<<<
-$word,"[" . $word ."]",$phonemes
@@ -104 +115 @@
-                $phonemes = join(" ",@line);
-                write MISSINGWORDSOUT;
+                write LOG;              
-        }
+        close MISSINGWORDSOUT;
+        close LOG;
-        return 1;
-}

Trunk/Scripts/Audio_scripts/AudioSegmentation/AudioBook/Text.pm

@@ -34 +34 @@
-####################################################################
-sub new {
-
+super,$
-        my %self;
-        $self{"inputfilename"} = $textFile;
+        $self{'log'} = $super->{'log'};         
-        _clean(\%self);
-        bless(\%self,$class);
@@ -49 +50 @@
-        my $debug = $self->{"debug"};   
-        my $inputfilename = $self->{"inputfilename"};
+        my $log = $self->{'log'};       
-
-        open(IN, "<$inputfilename") or confess ("error: cannot open input $inputfilename for input"); 
+        open(LOG,">>$log") or confess ("cannot open $log file");        
+        print LOG "Changes made to Text file\n";
+        print LOG   "-------------------------\n";      
-        my @eText = <IN>; # slurp in entire file into an array
-        my @words;
@@ -91 +96 @@
-                                        _processEmails($self, \@words, $word);
-                                #} elsif ($word =~ /(http|https|ftp):\/\/[\w\-.]+(:\d+)?\/[\w.\-~\/%]+/) { # url
-                   
+
-                                        _processUrls($self, \@words, $word);
-
+
+
-                                        _processNumbers ($self, \@words, $word);
-
+
+
+
+
+
+
-                                        $word =~ s/\.//g;
+                                        print LOG " $word\n";                           
-                                        push (@words, $word);                                   
-                                } else {
@@ -114 +126 @@
-                confess "error - fix: $word contains numbers\n"
-        } elsif ($word =~ /(\w+)\@(\w+)\.(\w+)/)  { # only catches basic email address
-
-
-
-
-
+
+
+
+
+
+
+
+
+
-        } else {
-                confess "error - fix: $word\n"
@@ -133 +149 @@
-        my ($self,$wordarray, $word)= @_;
-        my $debug = $self->{"debug"};
-
+
+
-        if ($word =~ /(www)?\.(\w+)\.([A-Za-z]{2,4})/)  { # URL: www.abc.com
-
-
+
+
+
+
+
-                push (@$wordarray, "DOT");                              
-
- $
+word
+$word
-                } else {
-
+word
-                }
-                push (@$wordarray, "DOT");      
-
+word
-        } elsif ($word =~ /(\w+)\.([A-Za-z]{2,4})/)  { # URL: abc.com
-                my $word1 = $1;
-                my $word2 = $2;
-"processingUrls: splitword
+LOG "changed URL $word to
-                if ($word1 =~ /\d+/) {
-;DOT;$word2;\n"
+ DOT $word2;\n" if $debug
-                        _processWordsContainingNumbers ($self, $wordarray,$word1);
-                } else {
@@ -168 +188 @@
-        if ($subword =~ /\d+\D+/) {  # assume single, consecutive set of numbers (i.e no split numbers in word)
-                my $number = $subword;
-
+ if $debug
-                $number =~ s/\D//g; # removes non-digit characters
-
+ if $debug
-                my @number = split(//, $number);
-                foreach my $digit (@number) {
-
+_
-                }               
-                my $word = $subword;
@@ -187 +207 @@
-                }       
-                my $number = $subword;
-
+ if $debug
-                $number =~ s/\D//g; # removes non-digit characters
-
+ if $debug
-                my @number = split(//, $number);
-                foreach my $digit (@number) {
-                        _processNumbers ($self, $wordarray, $digit); # will push the resulting ordinal converted to a word onto the array
-                }                       
-                
-        }       
-}
@@ -209 +228 @@
-                        push (@$words, uc $word);
-                }
-"info: 
+LOG "
-        } elsif (($number =~ /^\d{4}$/) and ($number < 2100)){ # assume 4 digit numbers between 1000 and 2100 are years
-                my $datenum = year2en($number);
@@ -217 +236 @@
-                        push (@$words, uc $word);
-                }       
-"info: 
+LOG "
-        } elsif ($number =~ /^\d+$/) { # contains only numbers
-                my $wordnum = num2en($number);
@@ -225 +244 @@
-                        push (@$words, uc $word);
-                }
-"info: 
+LOG "
-        } elsif ($number =~ /^£\d+/) { # convert pounds to words
-                $number =~ s/^£//; # remove pound sign
@@ -235 +254 @@
-                }
-                push (@$words, "POUNDS");
-"info: 
+LOG "
-        } elsif ($number =~ /^\$\d+/) { # convert dollars to words
-                $number =~ s/^\$//; # remove dollar sign
@@ -245 +264 @@
-                }
-                push (@$words, "DOLLARS");
-"info: 
+LOG "
-# skip this - minutes do not get processed properly (since they use a signle quote) - do manually in eText file.                
-#       } elsif ($number =~ /^\d{1,3}°$/) { # convert degrees to words
@@ -266 +285 @@
-                        push (@$words, uc $word);
-                }
-"info: 
+LOG "
-        } elsif ($number =~ /^\w+\d+/) { # convert ordinal number to words
-                my $numbertemp = $number;
@@ -276 +295 @@
-                        push (@$words, uc $word);
-                }
-"info: 
+LOG "
-        } else {
-                push (@$words, $number);

Trunk/Scripts/Audio_scripts/AudioSegmentation/AudioBook/output_files/prompts

@@ -1 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+