Changeset 2471

Timestamp:

03/20/08 12:05:43 (10 months ago)

Author:

kmaclean

Message:

VoxForge?_config.pm refactor to a class & add audio processing for German and Dutch audio files - snapshot

Files:

• Trunk/Scripts/Audio_scripts/AudioBooks/MissingWordsCleanup.pl (modified) (3 diffs)
• Trunk/Scripts/Audio_scripts/AudioBooks/fullrun.pl (added)
• Trunk/Scripts/Audio_scripts/Repository.pm (modified) (3 diffs)
• Trunk/Scripts/Audio_scripts/UserSubmission.pm (modified) (7 diffs)
• Trunk/Scripts/Mirroring_scripts/LocalUserSubmissionValidation.pl (modified) (1 diff)
• Trunk/Scripts/Mirroring_scripts/RunDaily.pl (modified) (3 diffs)
• Trunk/Scripts/Mirroring_scripts/RunDaily_preprocessing.pl (modified) (2 diffs)
• Trunk/Scripts/Mirroring_scripts/Single_WebGUIForum.pm (modified) (2 diffs)
• Trunk/Scripts/Mirroring_scripts/VoxForge_config.pm (modified) (3 diffs)
• Trunk/Scripts/Mirroring_scripts/VoxForge_config_testing.pm (modified) (7 diffs)
• Trunk/Scripts/Mirroring_scripts/etc/wgetGerman.sh (added)

Trunk/Scripts/Audio_scripts/AudioBooks/MissingWordsCleanup.pl

@@ -13 +13 @@
-### This program is free software; you can redistribute it and/or
-### modify it under the terms of the GNU General Public License
-3
+2
-### of the License, or (at your option) any later version.
-###
@@ -31 +31 @@
-my $debug =1;
-# Festival can't process some words with single quotes - these need to be hand-edited in the "MissingWords" File before running this program!
+open(MISSINGWORDS,"MissingWords") or die ("cannot open MissingWords file");
+my(@lines) = <MISSINGWORDS>; 
+close MISSINGWORDS;
+
+open(MISSINGWORDS,">MissingWords") or die ("cannot open MissingWords file");
+foreach my $line (@lines) {
+        if ($line =~ /\'/) {
+                my $formatedline = $line;
+                $formatedline =~ s/\n//g;
+                print "!!!!!!Check this word: before:$formatedline;";
+                $line =~ s/\'//g; # remove single quote 
+                print "after:$line";
+
+        } 
+        print MISSINGWORDS $line;
+}
+close MISSINGWORDS;
+
-my $festivalcommand = q|#!/usr/bin/festival --script
-(let
@@ -71 +89 @@
-        if (!($line =~ /festival>/)) {
-                print OUT "$filename    [$filename]    @line\n";        
+                print  "$filename    [$filename]    @line\n";
-        }
-}

Trunk/Scripts/Audio_scripts/Repository.pm

@@ -45 +45 @@
-sub Main {
-        my ($parms ) = @_;
-
-
-
-
-
-
+
+
+
+
+
+
+
+
+
+
-        my($dirlist) = ReadDir($ToBeProcessedDir); # $dirlist is a pointer to array created in ReadDir
-        my %ProcessedUserSubmissions;
-        foreach my $AudioDirName (@$dirlist) {  
-                my $readme = UserSubmission::README->new($parms, "$ToBeProcessedDir/$AudioDirName/etc/README");
-
+
+
+
+
-                $$parms{"AudioSubdirectory"} = $readme->getFiletype();  # set to wav or flac based on README contents
-                my $OriginalAudioRateDir = $readme->getSamplingrate() . "_" . $readme->getSamplingrateformat() ;
@@ -73 +80 @@
-                $$parms{"path_mfcc"} = "$AudioDirectory/MFCC/" . $$parms{"TargetAudioRateDir"} . "/" . $$parms{"HTK_ParameterKind"};
-                Repository::Main::Process($parms, $AudioDirName, $readme) || confess "Repository error: $?";            
-
-
-
+
+
+
+
+
+
-                # target 16kHz:16bit, MFCC_0_D          
-                my $samplingrate = $readme->getSamplingrate();
@@ -85 +95 @@
-                        Repository::Main::Process($parms, $AudioDirName, $readme) || confess "Repository error: $?";            
-                        Repository::Mfcc::Create($parms, $AudioDirName) || confess "Repository error: $?";
-
+
+
+
+
+
-                } else {
-                        print "$AudioDirectory audio is less than 16Khz.  No migration to Main/16kHz-16bit to avoid upsampling audio data\n"; 

Trunk/Scripts/Audio_scripts/UserSubmission.pm

@@ -57 +57 @@
-sub Main {
-        my ($parms ) = @_;
-
-
+
+
+
+
+
+
-        my($tarlist) = ReadTarDir($QuarantineDir); # $tarlist is a pointer to array created in ReadTarDir
-        my %directorynames;
@@ -79 +83 @@
-                }
-                # !!!!!!
-
-
-
-
-
-
-
-
-
-
-
-
+
+
+
+
+
+
+
+
+
-                }
-                # !!!!!!
@@ -114 +115 @@
-sub Convert2Unix {
-        my ($parms, $audio_dir_name ) = @_;
-
-
-
+
+
+
+
+
+
-        opendir(DIR, "$QuarantineDir/$audio_dir_name") || confess ("Unable to open directory: $QuarantineDir/$audio_dir_name\n");
-        while (my $filename = readdir(DIR)) {
@@ -169 +173 @@
-        #to do look at Archive::Extract Cpan Module to handle tar tgz tar.gz and zip extractions ...
-        my ($parms, $filename, $suffix, $tarfile) = @_;
-
-
-
-
+
+
+
+
+
+
-        my (@clamscan_output, $clamscan_results);
-        #$command = ("freshclam"); print "$command\n"; system($command) == 0 or confess "system $command failed: $?"; 
@@ -232 +238 @@
-sub ValidateFilenames {
-        my ($parms, $AudioDirName) = @_;
-
+
+
+
-        # check that user used PROMPTS & README for filenames
-        # windows defaults to prompts.txt
@@ -242 +250 @@
-
-sub Copy2ToBeProcessed {
-# !!!!!!
-        #my ($parms, $audio_dir_name, $filetype) = @_;
-        my ($parms, $audio_dir_name, $filetype, $audio) = @_;
-
-
-
-
-
-
+
+
+
+
+
+
+
+
-        $audio->Finalize($parms, $audio_dir_name, $filetype);
-        if ($audio->getError) {
-                confess "UserSubmission error: $audio->getErrorMessage" ;
-        }
-        # !!!!!!
-        $command = ("rm -rf $ToBeProcessedDir/$audio_dir_name");  print "$command\n" if $debug;  system($command)  ; 
-        $command = ("mkdir $ToBeProcessedDir/$audio_dir_name $ToBeProcessedDir/$audio_dir_name/etc");  print "$command\n" if $debug;  system($command)  ; 
-        # !!!!!!
-        #UserSubmission::AUDIO::Copy2ToBeProcessed($parms, $audio_dir_name, $filetype)  || confess "UserSubmission error: $?" ;
-        $audio->Copy2ToBeProcessed($parms, $audio_dir_name, $filetype);
-        if ($audio->getError) {
-                confess "UserSubmission error: $audio->getErrorMessage" ;
-        }       
-        # !!!!!!
-        if ($debug) {   
-                $command = ("cp -f $QuarantineDir/$audio_dir_name/* $ToBeProcessedDir/$audio_dir_name/etc");  print "$command\n"; system($command) ; # return value usually says "cp: omitting directory `/data/quarantine/granthulbert-ar-01032007/wav'"
@@ -283 +287 @@
-sub Cleanup {
-        my ($parms, $audio_dir_name) = @_;
-
-
-
-
-
-
-
+
+
+
+
+
+
+
+
+
+
+
-        return 1;
-}

Trunk/Scripts/Mirroring_scripts/LocalUserSubmissionValidation.pl

@@ -31 +31 @@
-#use VoxForge_config; 
-#my $parms = VoxForge_config::getParms;
-
-
+
+
+
+
+
-#overrides
-#$$parms{"testing"} = 1; # not required

Trunk/Scripts/Mirroring_scripts/RunDaily.pl

@@ -30 +30 @@
-### 0.1.7 - 2007.8.22 - add Flac audio processing
-### 0.1.8 - 2008.1.22 - addition of WebGUIForumUpdate module & convert README module to a class
+### 0.1.9 - 2008.3.20 - refactor VoxForge_config to a class
-####################################################################
-package RunDaily;
@@ -58 +59 @@
-
-#use VoxForge_config_testing; # for testing
-::getParms
+->new()
-use VoxForge_config; # no parenthesis in order to import %parm into this namespace
-
-
-
+
+
+
+
+
-my $command;
-####################################################################
-#  parms
-
-
-
-
-
-
-
-
-
-
+
+
+
+
+
+
+
+
+
+
+
+
-####################################################################
-### Main
-####################################################################
-
-
-
-
-
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
-Rsync() || confess "UserSubmissions error: $?";
-
+
+
-
-print STDOUT "!!!!!!run_daily completed\n";
@@ -97 +114 @@
-        print "\nRsync starting****************************************************\n\n";
-        # Mirror files to VoxForge repository
-
+
+
+
+
+
-        $command = ("rsync -r -v -t /data/svn-tarfiles/Main/Trunk -e ssh user\@voxforge1\.org\:downloads/Main --stats"); print "***$command\n";system($command) == 0 or confess "system $command failed: $?";  
-        $command = ("rsync -r -v -t /data/svn-tarfiles/Main/Tags -e ssh user\@voxforge1\.org\:downloads/Main --stats"); print "***$command\n";  system($command) == 0 or confess "system $command failed: $?"; 

Trunk/Scripts/Mirroring_scripts/RunDaily_preprocessing.pl

@@ -13 +13 @@
-### This program is free software; you can redistribute it and/or
-### modify it under the terms of the GNU General Public License
-3
+2
-### of the License, or (at your option) any later version.
-###
@@ -51 +51 @@
-        $command = ("svn update /data/svn-mirror/Main/Tags"); print "$command\n" if $debug;system($command) == 0 or die "system $command failed: $?"; 
-        $command = ("svn update /data/svn-mirror/SpeechCorpus/Trunk"); print "$command\n" if $debug;system($command) == 0 or die "system $command failed: $?"; 
+        $command = ("svn update /data/svn-mirror/Russian/Trunk"); print "$command\n" if $debug;system($command) == 0 or die "system $command failed: $?"; 
-# !!!!!!
-
+
+
+
-# !!!!!!
-        return 1;

Trunk/Scripts/Mirroring_scripts/Single_WebGUIForum.pm

@@ -30 +30 @@
-my $parms = VoxForge_config::getParms;
-
-
+ 
-
-use lib '../Audio_scripts';
@@ -36 +36 @@
-my %ProcessedUserSubmissions;
-my $command;
+my ($promptspath, $readmepath, $licensepath);
+my ($path_original,$AudioDirName,$tarfile);
-####################################################################
-### Directory
-####################################################################
-
-
-
-
-
-
-
-
-
-
-
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
-}
-print "tarfile2:$path_original/$tarfile\n";
-my @suffixlist = ("tgz", "tar\.gz", "zip");
-my ($AudioDirName,$path,$suffix) = fileparse($tarfile,@suffixlist);
-$AudioDirName =~ s/\.$//; # remove trailing period that fileparse does not remove for some reason??
-print "AudioDirName:$AudioDirName\n";
-
-my $ae = Archive::Extract->new( archive => "$path_original/$tarfile" );
-$ae->extract( to => "$path_original/$AudioDirName") or die $ae->error;;
-##########submission with tarfile as attachment:
-my $promptspath = "$path_original/$AudioDirName/prompts.txt";
-my $readmepath = "$path_original/$AudioDirName/readme.txt";
-my $licensepath = "$path_original/$AudioDirName/license.txt";
-
-##########fix submission that did not work properly; no tarfile to upload since linking to voxforge1.org
-#my $readmepath = "$path_original/$AudioDirName/etc/README";
-#my $promptspath = "$path_original/$AudioDirName/etc/prompts-original";
-#my $licensepath = "$path_original/$AudioDirName/LICENSE";
-#my $tarfile = undef;
-####################################################################
-### Main

Trunk/Scripts/Mirroring_scripts/VoxForge_config.pm

@@ -7 +7 @@
-### Date: 2007.04.23
-### Command: ./VoxForge_config.pm
-
+.1
-###             
-### Copyright (C) 2007 Ken MacLean
@@ -21 +21 @@
-### GNU General Public License for more details.
-###
+### 0.1.1 - 2008.3.20 - refactor to a class
-####################################################################
-package VoxForge_config;
-use POSIX qw(strftime);
-use strict;
+use diagnostics;
+use Carp;
+####################################################################
+### Class Variables
+####################################################################
-
+####################################################################
+### Constructor
+####################################################################
+sub new {
+        my ($class, $language) = @_;
+        my %parms;
+        my $self = \%parms;
+        bless($self,$class);
+        $self->getParms;
+        $self->setSpeechCorpusLanguage("EN"); #default = english
+        return $self;
+}
+
+####################################################################
+### Class Methods
+####################################################################
-sub getParms {
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
+
+
+
+
+
+
-
-        ####################################################################
@@ -52 +65 @@
-        my $mday = strftime "%d", localtime; #  %d      numeric day of the month, with leading zeros (eg 01..31)
-        
-
-       $parms{"Nightly_dirname"} = 
-       $parms{"NightlyAM_16kHz_16bit"} = 
-       $parms{"NightlyAM_8kHz_16bit"} = 
+$
+$$parms{"Nightly_dirname"} = $
+$$parms{"NightlyAM_16kHz_16bit"} = $
+$$parms{"NightlyAM_8kHz_16bit"} = $
-        ####################################################################
-        # Script Location
-$parms{"_Main"}
-       $parms{"QSdir"} = $parms{"_Main"}."
-       $parms{"ScriptsDirectory"} = $parms{"_Main"}."
-               $parms{"AudioScripts_dir"} = 
-                       $parms{"Repository_dir"} = 
-                       $parms{"UserSubmission_dir"} = 
-               $parms{"AcousticModel_scripts"} = 
-                       $parms{"HTK_dir"} = 
-               $parms{"MirroringScripts_dir"} = 
-                               $parms{"TarFiles_dir"} = 
-                               $parms{"directories2mirror"} = 
-               $parms{"Testing_scripts_dir"} = 
-                       $parms{"Nightly_test_dir"} = 
-               $parms{"Metrics_scripts_dir"} = 
+my $main
+$$parms{"QSdir"} = "$main
+$$parms{"ScriptsDirectory"} = "$main
+$$parms{"AudioScripts_dir"} = $
+$$parms{"Repository_dir"} = $
+$$parms{"UserSubmission_dir"} = $
+$$parms{"AcousticModel_scripts"} = $
+$$parms{"HTK_dir"} = $
+$$parms{"MirroringScripts_dir"} = $
+$$parms{"TarFiles_dir"} = $
+$$parms{"directories2mirror"} = $
+$$parms{"Testing_scripts_dir"} = $
+$$parms{"Nightly_test_dir"} = $
+$$parms{"Metrics_scripts_dir"} = $
-
-
-
-
+$
+$
+$
-        ####################################################################
-        # Acoustic Model Releases directory
-
-       $parms{"CurrentBuild_dir"} = 
-                       $parms{"CurrentAM_16kHz_16bit"} = 
-                       $parms{"CurrentAM_8kHz_16bit"} = 
+$
+$$parms{"CurrentBuild_dir"} = $
+$$parms{"CurrentAM_16kHz_16bit"} = $
+$$parms{"CurrentAM_8kHz_16bit"} = $
-        ####################################################################
-        # UserSubmission Location
-
-
-
+
+
-        ####################################################################
-        # NFS directory from AM create server server to WebGUI server
-
+$
-        ####################################################################
-        # HTK and Julian binary locations
-
-
-
+
+
+
+
+
-}               
+
+####################################################################
+### Settors
+####################################################################
+sub setSpeechCorpusLanguage {
+        my ($parms, $language ) = @_;
+        ####################################################################
+        # Speech Corpus Location
+        my $SpeechCorpus;
+        if ($language =~ /EN/) {
+                $SpeechCorpus  = "/data/svn-mirror/SpeechCorpus/Trunk";
+        } elsif ($language =~ /DE/) {
+                $SpeechCorpus = "/data/svn-mirror/Dutch/Trunk";
+        } elsif ($language =~ /NL/) {
+                $SpeechCorpus = "/data/svn-mirror/de";
+        } else {
+                confess "VoxForge_config error: no language selected\n";
+        }
+        
+        $$parms{"AudioDirectory"} = "$SpeechCorpus/Audio";
+        $$parms{"Main_8kHz_16bit"} = $$parms{"AudioDirectory"} . "/Main/8kHz_16bit";    # user metrics uses this
+        $$parms{"LexiconDirectory"} = "$SpeechCorpus/Lexicon";
+        $$parms{"PromptsFile_16kHz-16bit"} = "$SpeechCorpus/Prompts/master_prompts_16kHz-16bit"; 
+        $$parms{"PromptsFile_8kHz-16bit"} = "$SpeechCorpus/Prompts//master_prompts_8kHz-16bit"; 
+        ####################################################################
+        # Speech Corpus Releases directory - none yet ...
+        
+}
+####################################################################
+### Gettors
+####################################################################
+sub getTesting {
+        my $self = shift;
+        return $self->{"testing"};
+}
+
+sub getDebug {
+        my $self = shift;
+        return $self->{"debug"};
+}
+
+sub getQuarantineDir {
+        my $self = shift;
+        return $self->{"QuarantineDir"};
+}
+
+sub getToBeProcessedDir {
+        my $self = shift;
+        return $self->{"ToBeProcessedDir"};
+}
+
+sub getAudioDirectory {
+        my $self = shift;
+        return $self->{"AudioDirectory"};
+}
+
+sub getMain_8kHz_16bit {
+        my $self = shift;
+        return $self->{"Main_8kHz_16bit"};
+}
+
+sub getMain_16kHz_16bit {
+        my $self = shift;
+        return $self->{"Main_16kHz_16bit"};
+}
+
+sub getLexiconDirectory {
+        my $self = shift;
+        return $self->{"LexiconDirectory"};
+}
+
+sub getPromptsFile_16kHz_16bit {
+        my $self = shift;
+        return $self->{"PromptsFile_16kHz-16bit"};
+}
+
+sub getPromptsFile_8kHz_16bit {
+        my $self = shift;
+        return $self->{"PromptsFile_8kHz-16bit"};
+}
-1;      

Trunk/Scripts/Mirroring_scripts/VoxForge_config_testing.pm

@@ -33 +33 @@
-        my ($class, $language) = @_;
-        my %parms;
-        getParms(\%parms);
-        setSpeechCorpusLanguage(\%parms, "EN");
-        my $self = \%parms;
-        bless($self,$class);
+        $self->getParms;
+        $self->setSpeechCorpusLanguage("EN"); #default = english
-        return $self;
-}
@@ -45 +45 @@
-        # !!!!!!
-        #my %parms;
-($parms ) = @_
+$parms = shift
-        # !!!!!
-        $$parms{"testing"} = 0; 
@@ -66 +66 @@
-        ####################################################################
-        # Script Location
-$$parms{"_Main"}
-       $$parms{"QSdir"} = $$parms{"_Main"}."
-       $$parms{"ScriptsDirectory"} = $$parms{"_Main"}."
-               
-                       
-                       
-               
-                       
-               
-                               
-                               
-               
-                       
-               
+my $main
+$$parms{"QSdir"} = "$main
+$$parms{"ScriptsDirectory"} = "$main
+
+
+
+
+
+
+
+
+
+
+
-
-#       $$parms{"directories2mirror"} = "TarFiles/Audio/directories2mirror-test";               # for testing
@@ -87 +87 @@
-        # Speech Corpus Releases directory - none yet ...
-        $$parms{"Tags"}  = "/data/svn-mirror/Main/Tags/Releases/";
-       
-                       
-                       
+
+
+
-        ####################################################################
-        # UserSubmission Location
@@ -119 +119 @@
-        } elsif ($language =~ /NL/) {
-                $SpeechCorpus = "/home/kmaclean/VoxForge-dev/Dutch";
+        } else {
+                confess "VoxForge_config error: no language selected\n";
-        }
-                
@@ -132 +134 @@
-### Gettors
-####################################################################
+sub getTesting {
+        my $self = shift;
+        return $self->{"testing"};
+}
+
+sub getDebug {
+        my $self = shift;
+        return $self->{"debug"};
+}
+
-sub getQuarantineDir {
-        my $self = shift;
-        return $self->{"Quarantine