Changeset 2263
- Timestamp:
- 08/23/07 15:16:30 (1 year ago)
- Files:
-
- Trunk/Scripts/Audio_scripts/AudioBooks/AudioBook.pl (modified) (1 diff)
- Trunk/Scripts/Audio_scripts/AudioBooks/htksegment.pl (modified) (1 diff)
- Trunk/Scripts/Audio_scripts/Repository.pm (modified) (4 diffs)
- Trunk/Scripts/Audio_scripts/Repository/Main.pm (modified) (4 diffs)
- Trunk/Scripts/Audio_scripts/Repository/Mfcc (moved) (moved from Trunk/Scripts/Audio_scripts/Repository/MFCC)
- Trunk/Scripts/Audio_scripts/Repository/Mfcc.pm (moved) (moved from Trunk/Scripts/Audio_scripts/Repository/MFCC.pm) (5 diffs)
- Trunk/Scripts/Audio_scripts/Repository/Original.pm (modified) (3 diffs)
- Trunk/Scripts/Audio_scripts/Repository/archive/downsample.pl (moved) (moved from Trunk/Scripts/Audio_scripts/Repository/Main/downsample.pl) (1 diff)
- Trunk/Scripts/Audio_scripts/UserSubmission.pm (modified) (16 diffs)
- Trunk/Scripts/Audio_scripts/UserSubmission/AUDIO (added)
- Trunk/Scripts/Audio_scripts/UserSubmission/AUDIO.pm (modified) (8 diffs)
- Trunk/Scripts/Audio_scripts/UserSubmission/AUDIO/Audio2PromptsValidation.sh (moved) (moved from Trunk/Scripts/Audio_scripts/UserSubmission/Audio2PromptsValidation.sh) (1 diff)
- Trunk/Scripts/Audio_scripts/UserSubmission/LICENSE.pm (modified) (2 diffs)
- Trunk/Scripts/Audio_scripts/UserSubmission/PROMPTS.pm (modified) (1 diff)
- Trunk/Scripts/Audio_scripts/UserSubmission/Prompts2codetrainscp.pl (modified) (1 diff)
- Trunk/Scripts/Audio_scripts/UserSubmission/README.pm (modified) (2 diffs)
- Trunk/Scripts/Mirroring_scripts/LocalUserSubmissionValidation.pl (modified) (2 diffs)
- Trunk/Scripts/Mirroring_scripts/RunDaily.pl (modified) (2 diffs)
Legend:
- Unmodified
- Added
- Removed
- Modified
- Copied
- Moved
Trunk/Scripts/Audio_scripts/AudioBooks/AudioBook.pl
r2229 r2263 238 238 $line =~ s/\(//g; # remove parenthesis 239 239 $line =~ s/\)//g; # remove parenthesis 240 $line =~ s/\[//g; # remove bracket 241 $line =~ s/\]//g; # remove bracket 240 242 $line =~ s/_//g; # remove underscore 243 $line =~ s/\*//g; # remove asterisk 241 244 # Other cleanup 242 245 $line =~ s/&/AND/g; Trunk/Scripts/Audio_scripts/AudioBooks/htksegment.pl
r2224 r2263 143 143 my @aligned_line = split(/ /,$aligned_words[$sentence_end + $increment]); 144 144 my ($word,$startTime,$endTime,$pause) = @aligned_line; 145 if ($pause > $min_pause_for_sentence_break) {145 if ($pause >= $min_pause_for_sentence_break) { 146 146 my $count; 147 147 my $padded_fileid = sprintf("%04d",$fileid); Trunk/Scripts/Audio_scripts/Repository.pm
r2189 r2263 6 6 ### email: contact@voxforge.org 7 7 ### Date: 2006.09.26 8 ### Version: 0. 38 ### Version: 0.4 9 9 ### 10 10 ### Copyright (C) 2006 Ken MacLean … … 22 22 ### Change History: 23 23 ### rel0.3 - 2007.01.9 - modularize code and updates as part of user submission automation 24 ### 24 ### r0.4 (Aug 22, 2007) - added Flac processing 25 25 #################################################################### 26 26 package Repository; 27 27 use strict; 28 28 use File::Spec; 29 use diagnostics; 30 use Carp; 29 31 30 32 # modules 31 33 use UserSubmission::README(); 32 34 use Repository::Main(); 33 use Repository::M FCC();35 use Repository::Mfcc(); 34 36 use Repository::Original(); 35 37 use UserSubmission::PROMPTS(); … … 64 66 ($$parms{"TargetAudioRateDir"}, $$parms{"DownsampleRate"}, $$parms{"DownsampleBitsPerSample"}) = ("8kHz_16bit", 8000, 16); 65 67 $$parms{"path_main"} = "$AudioDirectory/Main/" . $$parms{"TargetAudioRateDir"}; 66 $$parms{"path_mfcc"} = "$AudioDirectory/M FCC/" . $$parms{"TargetAudioRateDir"} . "/" . $$parms{"HTK_ParameterKind"};68 $$parms{"path_mfcc"} = "$AudioDirectory/Mfcc/" . $$parms{"TargetAudioRateDir"} . "/" . $$parms{"HTK_ParameterKind"}; 67 69 Repository::Main::Process($parms, \%AudioDirParms) || die "Repository error: $?"; 68 Repository::M FCC::Create($parms, \%AudioDirParms) || die "Repository error: $?";70 Repository::Mfcc::Create($parms, \%AudioDirParms) || die "Repository error: $?"; 69 71 UserSubmission::PROMPTS::UpdateMasterPrompts($parms, \%AudioDirParms,$$parms{"PromptsFile_8kHz-16bit"} ) || die "Repository error: $?"; 70 72 … … 75 77 ($$parms{"TargetAudioRateDir"}, $$parms{"DownsampleRate"}, $$parms{"DownsampleBitsPerSample"}) = ("16kHz_16bit", 16000, 16); 76 78 $$parms{"path_main"} = "$AudioDirectory/Main/" . $$parms{"TargetAudioRateDir"}; 77 $$parms{"path_mfcc"} = "$AudioDirectory/M FCC/" . $$parms{"TargetAudioRateDir"} . "/" . $$parms{"HTK_ParameterKind"};79 $$parms{"path_mfcc"} = "$AudioDirectory/Mfcc/" . $$parms{"TargetAudioRateDir"} . "/" . $$parms{"HTK_ParameterKind"}; 78 80 Repository::Main::Process($parms, \%AudioDirParms) || die "Repository error: $?"; 79 Repository::M FCC::Create($parms, \%AudioDirParms) || die "Repository error: $?";81 Repository::Mfcc::Create($parms, \%AudioDirParms) || die "Repository error: $?"; 80 82 UserSubmission::PROMPTS::UpdateMasterPrompts($parms, \%AudioDirParms,$$parms{"PromptsFile_16kHz-16bit"} ) || die "Repository error: $?"; 81 83 } else { Trunk/Scripts/Audio_scripts/Repository/Main.pm
r2196 r2263 3 3 ### 4 4 ### script name : Main.pm 5 ### version: 0. 15 ### version: 0.2 6 6 ### created by: Ken MacLean 7 7 ### mail: contact@voxforge.org … … 12 12 ### This program is free software; you can redistribute it and/or 13 13 ### modify it under the terms of the GNU General Public License 14 ### as published by the Free Software Foundation; either version 214 ### as published by the Free Software Foundation; either version 3 15 15 ### of the License, or (at your option) any later version. 16 16 ### … … 19 19 ### MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 20 20 ### GNU General Public License for more details. 21 ### 21 ### 22 ### r0.2 (Aug 22, 2007) - added Flac processing 22 23 #################################################################### 23 24 package Repository::Main; 24 25 use strict; 26 use diagnostics; 27 use Carp; 28 # modules 29 use lib '../../Audio_scripts'; 30 use UserSubmission::AUDIO(); 31 #################################################################### 25 32 my $command; 26 33 … … 39 46 40 47 # copy audio files from Original directory to Main directory 41 $command = ("svn copy $path_original/$audio_dir_name $path_main/$audio_dir_name"); print "$command\n" if $debug; system($command) == 0 or die "system $command failed: $?"; 48 $command = ("svn copy $path_original/$audio_dir_name $path_main/$audio_dir_name"); print "$command\n" if $debug; system($command) == 0 or confess "system $command failed: $?"; 49 # !!!!!! 42 50 # run downsample script #./downsample.pl will downsample files and keep original file suffix - will not rename 'raw' to 'wav' file suffix 43 $command = ("cd $AudioScripts_dir/Repository && ./Main/downsample.pl $path_main/$audio_dir_name/$audio_subdirectory $OriginalSamplingRate $downsample_rate $debug");print "$command\n" if $debug; system($command) == 0 or die "system $command failed: $?"; 51 #$command = ("cd $AudioScripts_dir/Repository && ./Main/downsample.pl $path_main/$audio_dir_name/$audio_subdirectory $OriginalSamplingRate $downsample_rate $debug");print "$command\n" if $debug; system($command) == 0 or confess "system $command failed: $?"; 52 UserSubmission::AUDIO::Downsample($parms, $AudioDirParms) || confess "Repository error: $?"; 53 # !!!!!! 44 54 # update properties 45 $command = ("svn propset -R sampling_rate_Hz $downsample_rate $path_main/$audio_dir_name"); print "$command\n" if $debug; system($command) == 0 or die"system $command failed: $?";46 $command = ("svn propset -R bits_per_sample $downsample_bits_per_sample $path_main/$audio_dir_name" ); print "$command\n" if $debug; system($command) == 0 or die"system $command failed: $?";47 $command = ("svn update -r HEAD $path_main/$audio_dir_name");print "$command\n" if $debug; system($command) == 0 or die"system $command failed: $?";48 $command = ("svn commit -m \"downsampled $audio_dir_name audio files & updated properties\" $path_main/$audio_dir_name"); print "$command\n" if $debug; system($command) == 0 or die"system $command failed: $?";55 $command = ("svn propset -R sampling_rate_Hz $downsample_rate $path_main/$audio_dir_name"); print "$command\n" if $debug; system($command) == 0 or confess "system $command failed: $?"; 56 $command = ("svn propset -R bits_per_sample $downsample_bits_per_sample $path_main/$audio_dir_name" ); print "$command\n" if $debug; system($command) == 0 or confess "system $command failed: $?"; 57 $command = ("svn update -r HEAD $path_main/$audio_dir_name");print "$command\n" if $debug; system($command) == 0 or confess "system $command failed: $?"; 58 $command = ("svn commit -m \"downsampled $audio_dir_name audio files & updated properties\" $path_main/$audio_dir_name"); print "$command\n" if $debug; system($command) == 0 or confess "system $command failed: $?"; 49 59 print "Main completed for $path_original/$audio_dir_name\n"; 50 60 return 1; Trunk/Scripts/Audio_scripts/Repository/Mfcc.pm
r2196 r2263 2 2 #################################################################### 3 3 ### 4 ### script name : M FCC.pm5 ### version: 0. 14 ### script name : Mfcc.pm 5 ### version: 0.2 6 6 ### created by: Ken MacLean 7 7 ### mail: contact@voxforge.org … … 12 12 ### This program is free software; you can redistribute it and/or 13 13 ### modify it under the terms of the GNU General Public License 14 ### as published by the Free Software Foundation; either version 214 ### as published by the Free Software Foundation; either version 3 15 15 ### of the License, or (at your option) any later version. 16 16 ### … … 21 21 ### 22 22 ### Change log: 23 ### Release 0.1.1 jan 22, 2006 - integrated ConvertWav2mfc.pl script 23 ### Release 0.1.1 jan 22, 2006 - integrated ConvertWav2mfc.pl script 24 ### r0.2 (Aug 22, 2007) - added Flac processing 24 25 ### 25 26 #################################################################### 26 package Repository::M FCC;27 package Repository::Mfcc; 27 28 use strict; 29 use diagnostics; 30 use Carp; 28 31 my $command; 29 32 … … 40 43 41 44 #cleanup required before copy (cleanup .svn files???) 42 $command = ("svn cleanup $path_main/$audio_dir_name" ); print "$command\n" if $debug; system($command) == 0 or die"system $command failed: $?";45 $command = ("svn cleanup $path_main/$audio_dir_name" ); print "$command\n" if $debug; system($command) == 0 or confess "system $command failed: $?"; 43 46 # make sure target directory exists (i.e. 'MFCC/16kHz:16-bit') 44 47 # copy audio files from Original directory to Main directory 45 $command = ("svn copy $path_main/$audio_dir_name $path_mfcc/$audio_dir_name" ); print "$command\n" if $debug; system($command) == 0 or die"system $command failed: $?";46 $command = ("svn update -r HEAD $path_mfcc/$audio_dir_name");print "$command\n" if $debug; system($command) == 0 or die"system $command failed: $?";47 $command = ("svn commit -m \"copied $audio_dir_name wav files to MFCC directory\" $path_mfcc/$audio_dir_name");print "$command\n" if $debug; system($command) == 0 or die"system $command failed: $?";48 $command = ("svn copy $path_main/$audio_dir_name $path_mfcc/$audio_dir_name" ); print "$command\n" if $debug; system($command) == 0 or confess "system $command failed: $?"; 49 $command = ("svn update -r HEAD $path_mfcc/$audio_dir_name");print "$command\n" if $debug; system($command) == 0 or confess "system $command failed: $?"; 50 $command = ("svn commit -m \"copied $audio_dir_name wav files to MFCC directory\" $path_mfcc/$audio_dir_name");print "$command\n" if $debug; system($command) == 0 or confess "system $command failed: $?"; 48 51 # rename wav directory to mfc 49 $command = ("svn move $path_mfcc/$audio_dir_name/$audio_subdirectory $path_mfcc/$audio_dir_name/mfc"); print "$command\n" if $debug; system($command) == 0 or die"system $command failed: $?";50 $command = ("svn update -r HEAD $path_mfcc/$audio_dir_name"); print "$command\n" if $debug; system($command) == 0 or die"system $command failed: $?";51 $command = ("svn commit -m \"renamed $audio_dir_name wav directory to mfc\" $path_mfcc/$audio_dir_name"); print "$command\n" if $debug; system($command) == 0 or die"system $command failed: $?";52 $command = ("svn move $path_mfcc/$audio_dir_name/$audio_subdirectory $path_mfcc/$audio_dir_name/mfc"); print "$command\n" if $debug; system($command) == 0 or confess "system $command failed: $?"; 53 $command = ("svn update -r HEAD $path_mfcc/$audio_dir_name"); print "$command\n" if $debug; system($command) == 0 or confess "system $command failed: $?"; 54 $command = ("svn commit -m \"renamed $audio_dir_name wav directory to mfc\" $path_mfcc/$audio_dir_name"); print "$command\n" if $debug; system($command) == 0 or confess "system $command failed: $?"; 52 55 #Convert wav audio to HTK mfcc script 53 56 # do not refresh in Eclipse (i.e. Subclipse client to Subversion) to view progress of this script 54 57 # refreshing svn workspace causes strange errors with this script!!! 55 58 # !!!!!! 56 # $command = ("cd $AudioScripts_dir/Repository && ./ConvertWav2mfc.pl $path_mfcc/$audio_dir_name/mfc $debug"); print "$command\n" if $debug; system($command) == 0 or die"system $command failed: $?";57 ConvertWav2mfc($parms, $AudioDirParms)|| die"MFCC\.pm error: $?";59 # $command = ("cd $AudioScripts_dir/Repository && ./ConvertWav2mfc.pl $path_mfcc/$audio_dir_name/mfc $debug"); print "$command\n" if $debug; system($command) == 0 or confess "system $command failed: $?"; 60 ConvertWav2mfc($parms, $AudioDirParms)|| confess "MFCC\.pm error: $?"; 58 61 # !!!!!! 59 62 #update properties 60 $command = ("svn propset -R HTK_ParameterKindWithQualifiers $HTK_ParameterKindWithQualifiers $path_mfcc/$audio_dir_name"); print "$command\n" if $debug; system($command) == 0 or die"system $command failed: $?";61 $command = ("svn update -r HEAD $path_mfcc"); print "$command\n" if $debug; system($command) == 0 or die"system $command failed: $?";62 $command = ("svn commit -m \"$audio_dir_name wav files converted to HTK mfc & updated properties\" $path_mfcc/$audio_dir_name"); print "$command\n" if $debug; system($command) == 0 or die"system $command failed: $?";63 $command = ("svn propset -R HTK_ParameterKindWithQualifiers $HTK_ParameterKindWithQualifiers $path_mfcc/$audio_dir_name"); print "$command\n" if $debug; system($command) == 0 or confess "system $command failed: $?"; 64 $command = ("svn update -r HEAD $path_mfcc"); print "$command\n" if $debug; system($command) == 0 or confess "system $command failed: $?"; 65 $command = ("svn commit -m \"$audio_dir_name wav files converted to HTK mfc & updated properties\" $path_mfcc/$audio_dir_name"); print "$command\n" if $debug; system($command) == 0 or confess "system $command failed: $?"; 63 66 return 1; 64 67 } … … 76 79 77 80 # open directory 78 opendir(DIR, "$directory") || die("Unable to open directory: $directory");81 opendir(DIR, "$directory") || confess ("Unable to open directory: $directory"); 79 82 80 83 # process each file name in directory one at a time 81 84 while (my $filename = readdir(DIR)) { 82 85 chomp ($filename); 83 if ($filename ne "." and $filename ne ".." ) { 84 my($temp, $suffix1) =split(/\./, $filename); # this tells you that there is a file name that has at least one period in its name 85 if (defined($suffix1)) { # this is to address the cases where a file has no suffix 86 my @filename = split(/\./,$filename); 87 my $suffix = pop(@filename); # to address the case where there is more than one period in the file name 88 my $filename_nosuffix = $filename ; 89 $filename_nosuffix =~ s/\.$suffix//; 86 if ($filename ne "." and $filename ne ".." and $filename ne ".svn") { 87 # !!!!!! 88 my @suffixlist = ("wav", "raw", "flac"); 89 my ($filename_nosuffix, $path, $suffix) = fileparse($filename, @suffixlist); 90 #my($temp, $suffix1) =split(/\./, $filename); # this tells you that there is a file name that has at least one period in its name 91 #if (defined($suffix1)) { # this is to address the cases where a file has no suffix 92 #my @filename = split(/\./,$filename); 93 #my $suffix = pop(@filename); # to address the case where there is more than one period in the file name 94 #my $filename_nosuffix = $filename ; 95 #$filename_nosuffix =~ s/\.$suffix//; 90 96 print "filename:$filename;filename_nosuffix:$filename_nosuffix;suffix:$suffix\n"; 97 # !!!!!! 91 98 if ($suffix eq "wav") { 99 print "converting:$directory/$filename\n" if $debug; 100 # subversion - rename file in Working Copy (need to commit these changes to repository once you are completed) 101 $command = ("svn rename $directory/$filename $directory/$filename_nosuffix\.mfc" ); system($command) == 0 or confess "system $command failed: $?"; 102 sleep(3); # wait svn working copy to complete processing ??? 103 # HTK - convert wav file to HTK mfc format 104 $command = ("$HTKbin/HCopy -A -D -T 1 -C $Repository_dir/MFCC/wav_config $directory/$filename_nosuffix\.mfc $directory/$filename_nosuffix\.mfc" ); system($command) == 0 or confess "system $command failed: $?"; 105 } elsif ($suffix eq "flac") { 92 106 print "converting:$directory\/$filename\n" if $debug; 93 107 # subversion - rename file in Working Copy (need to commit these changes to repository once you are completed) 94 $command = ("svn rename $directory/$filename $directory/$filename_nosuffix\.mfc" ); system($command) == 0 or die "system $command failed: $?"; 108 # flac to wav 109 $command = ("flac -d $directory/$filename"); print "$command\n" if $debug; system($command) == 0 or confess confess "system $command failed: $?"; 110 # remove flac files 111 $command = ("rm -f $directory/$filename"); print "$command\n" if $debug; system($command) == 0 or confess confess "system $command failed: $?"; 112 $command = ("svn rename $directory/$filename_nosuffix.wav $directory/$filename_nosuffix\.mfc" ); system($command) == 0 or confess "system $command failed: $?"; 95 113 sleep(3); # wait svn working copy to complete processing ??? 96 114 # HTK - convert wav file to HTK mfc format 97 $command = ("$HTKbin/HCopy -A -D -T 1 -C $Repository_dir/MFCC/wav_config $directory/$filename_nosuffix\.mfc $directory/$filename_nosuffix\.mfc" ); system($command) == 0 or die"system $command failed: $?";115 $command = ("$HTKbin/HCopy -A -D -T 1 -C $Repository_dir/MFCC/wav_config $directory/$filename_nosuffix\.mfc $directory/$filename_nosuffix\.mfc" ); system($command) == 0 or confess "system $command failed: $?"; 98 116 } 99 }100 117 } 101 118 } Trunk/Scripts/Audio_scripts/Repository/Original.pm
r2158 r2263 12 12 ### This program is free software; you can redistribute it and/or 13 13 ### modify it under the terms of the GNU General Public License 14 ### as published by the Free Software Foundation; either version 214 ### as published by the Free Software Foundation; either version 3 15 15 ### of the License, or (at your option) any later version. 16 16 ### … … 23 23 package Repository::Original; 24 24 use strict; 25 use diagnostics; 26 use Carp; 27 25 28 my $command; 26 29 … … 38 41 # copy ToBeProcessed directory to Working Copy 39 42 $command = ("mkdir $path_original/$audio_dir_name"); print "$command\n" if $debug; system($command) ; 40 $command = ("cp -r $ToBeProcessedDir/$audio_dir_name $path_original"); print "$command\n" if $debug; system($command) == 0 or die"system $command failed: $?";43 $command = ("cp -r $ToBeProcessedDir/$audio_dir_name $path_original"); print "$command\n" if $debug; system($command) == 0 or confess "system $command failed: $?"; 41 44 # add to Working Copy and then add to Repository 42 $command = ("svn add $path_original/$audio_dir_name"); print "$command\n" if $debug; system($command) == 0 or die"system $command failed: $?";43 $command = ("svn update -r HEAD $path_original/$audio_dir_name"); print "$command\n" if $debug; system($command) == 0 or die"system $command failed: $?";44 $command = ("svn commit -m \"commit $audio_dir_name audio files\" $path_original/$audio_dir_name");print "$command\n" if $debug; system($command) == 0 or die"system $command failed: $?";45 $command = ("svn add $path_original/$audio_dir_name"); print "$command\n" if $debug; system($command) == 0 or confess "system $command failed: $?"; 46 $command = ("svn update -r HEAD $path_original/$audio_dir_name"); print "$command\n" if $debug; system($command) == 0 or confess "system $command failed: $?"; 47 $command = ("svn commit -m \"commit $audio_dir_name audio files\" $path_original/$audio_dir_name");print "$command\n" if $debug; system($command) == 0 or confess "system $command failed: $?"; 45 48 # update properties 46 $command = ("svn propset -R LICENSE -F $path_original/$audio_dir_name/LICENSE $path_original/$audio_dir_name" ); print "$command\n" if $debug; system($command) == 0 or die"system $command failed: $?";47 $command = ("svn propset -R README -F $path_original/$audio_dir_name/etc/README $path_original/$audio_dir_name" ); print "$command\n" if $debug; system($command) == 0 or die"system $command failed: $?";48 $command = ("svn propset -R sampling_rate_Hz $OriginalSamplingRate $path_original/$audio_dir_name" ); print "$command\n" if $debug; system($command) == 0 or die"system $command failed: $?";49 $command = ("svn propset -R bits_per_sample $OriginalBitsPerSample $path_original/$audio_dir_name" ); print "$command\n" if $debug; system($command) == 0 or die"system $command failed: $?";50 $command = ("svn propset -R dialect \"$dialect\" $path_original/$audio_dir_name" ); print "$command\n" if $debug; system($command) == 0 or die"system $command failed: $?";51 $command = ("svn update -r HEAD $path_original/$audio_dir_name"); print "$command\n" if $debug; system($command) == 0 or die"system $command failed: $?";52 $command = ("svn commit -m \"add properties for $audio_dir_name audio files\" $path_original/$audio_dir_name");print "$command\n" if $debug; system($command) == 0 or die"system $command failed: $?";49 $command = ("svn propset -R LICENSE -F $path_original/$audio_dir_name/LICENSE $path_original/$audio_dir_name" ); print "$command\n" if $debug; system($command) == 0 or confess "system $command failed: $?"; 50 $command = ("svn propset -R README -F $path_original/$audio_dir_name/etc/README $path_original/$audio_dir_name" ); print "$command\n" if $debug; system($command) == 0 or confess "system $command failed: $?"; 51 $command = ("svn propset -R sampling_rate_Hz $OriginalSamplingRate $path_original/$audio_dir_name" ); print "$command\n" if $debug; system($command) == 0 or confess "system $command failed: $?"; 52 $command = ("svn propset -R bits_per_sample $OriginalBitsPerSample $path_original/$audio_dir_name" ); print "$command\n" if $debug; system($command) == 0 or confess "system $command failed: $?"; 53 $command = ("svn propset -R dialect \"$dialect\" $path_original/$audio_dir_name" ); print "$command\n" if $debug; system($command) == 0 or confess "system $command failed: $?"; 54 $command = ("svn update -r HEAD $path_original/$audio_dir_name"); print "$command\n" if $debug; system($command) == 0 or confess "system $command failed: $?"; 55 $command = ("svn commit -m \"add properties for $audio_dir_name audio files\" $path_original/$audio_dir_name");print "$command\n" if $debug; system($command) == 0 or confess "system $command failed: $?"; 53 56 print "Original completed for $path_original/$audio_dir_name\n"; 54 57 return 1; Trunk/Scripts/Audio_scripts/Repository/archive/downsample.pl
r2198 r2263 23 23 #################################################################### 24 24 use strict; 25 use File::Spec; 25 26 26 27 27 my ($targetrate, @command); Trunk/Scripts/Audio_scripts/UserSubmission.pm
r2220 r2263 3 3 ### 4 4 ### script name : UserSubmission.pl 5 ### version: 0. 15 ### version: 0.2 6 6 ### created by: Ken MacLean 7 7 ### mail: contact@voxforge.org … … 13 13 ### This program is free software; you can redistribute it and/or 14 14 ### modify it under the terms of the GNU General Public License 15 ### as published by the Free Software Foundation; either version 215 ### as published by the Free Software Foundation; either version 3 16 16 ### of the License, or (at your option) any later version. 17 17 ### … … 20 20 ### MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 21 21 ### GNU General Public License for more details. 22 ### 22 ### 23 ### r0.2 (Aug 22, 2007) - added Flac processing 23 24 #################################################################### 24 25 package UserSubmission; 25 26 use strict; 26 27 use File::Basename; 27 use Audio::Wav; 28 use diagnostics; 29 use Carp; 28 30 # internal modules perlmod 29 31 #use UserSubmission::README; # Import @EXPORT symbols … … 59 61 foreach my $tarfile (@$tarlist) { 60 62 my %AudioDirParms; 61 # !!!!!!62 #my($audio_dir_name, $suffix) =split(/\./, $tarfile);63 #($AudioDirParms {"audio_dir_name"}, $AudioDirParms {"suffix1"} , $AudioDirParms {"suffix2"}) = split(/\./, $tarfile);64 63 my @suffixlist = ("tgz", "tar\.gz", "zip"); 65 64 my ($audio_dir_name,$path); … … 68 67 $AudioDirParms {"audio_dir_name"} = $audio_dir_name; 69 68 70 #my @tarfile = split(/\./, $tarfile);71 #$AudioDirParms {"suffix2"} = pop(@tarfile);72 #$AudioDirParms {"suffix1"} = pop(@tarfile);73 #$AudioDirParms {"audio_dir_name"} = "@tarfile"; # assumes there are no other periods in the file name!74 #if (!defined($AudioDirParms {"suffix1"})) {$AudioDirParms {"suffix1"}=""};75 #if (!defined($AudioDirParms {"suffix2"})) {$AudioDirParms {"suffix2"}=""};76 #if (($AudioDirParms {"suffix2"} eq "tgz") and ($AudioDirParms {"suffix1"})) { # assumes a VoxForgeIVR submissions77 # $AudioDirParms {"audio_dir_name"} = $AudioDirParms {"audio_dir_name"} . "\." . $AudioDirParms {"suffix1"};78 # $AudioDirParms {"suffix1"} = "";79 # }80 #my $audio_dir_name = $AudioDirParms {"audio_dir_name"};81 # !!!!!!82 69 $command = ("rm -rf $QuarantineDir/$audio_dir_name"); print "$command\n" if $debug; system($command); # remove ToBeProcessed directory - if it exists 83 runClamAV($parms, \%AudioDirParms, $tarfile) || die"UserSubmission error: $?" ; # runs ClamAV and extracts gzipped tar or Zip file70 runClamAV($parms, \%AudioDirParms, $tarfile) || confess "UserSubmission error: $?" ; # runs ClamAV and extracts gzipped tar or Zip file 84 71 Convert2Unix($parms, \%AudioDirParms); 85 ValidateFilenames($parms, \%AudioDirParms) || die "UserSubmission error: $?" ; 86 ($AudioDirParms{"samplingrate"}, $AudioDirParms{"samplingrateformat"}, $AudioDirParms{"filetype"}, $AudioDirParms{"dialect"}) = UserSubmission::README::Process($parms, \%AudioDirParms); # can't put "|| die "UserSubmission error: $?"" becuase changes this to scalar context, and only last item in list is transferred! 87 print "UserSubmission-README says:".$AudioDirParms{"samplingrate"}."-".$AudioDirParms{"samplingrateformat"}."-".$AudioDirParms{"filetype"}.";dialect:".$AudioDirParms{"dialect"}."\n\n" if $debug; 88 UserSubmission::AUDIO::WavValidation($parms, \%AudioDirParms) || die "UserSubmission error: $?" ; #checks that audio rates in README match actual files 89 UserSubmission::PROMPTS::Clean($parms, \%AudioDirParms) || die "UserSubmission error: $?"; # creates reformatted prompts file for processing with HVite and later merging into Master_Prompts 90 UserSubmission::PROMPTS::Validation($parms, \%AudioDirParms) || die "UserSubmission error: $?"; # use HDMan to run prompts against VoxForge Dictionnary 91 UserSubmission::AUDIO::Audio2PromptsValidation($parms, \%AudioDirParms) || die "UserSubmission error: $?" ; # partial AM training session using audio data to see if HVite has any problems trying to re-align the data - if it does, then audio does not match transcription 92 $directorynames{$audio_dir_name} = [$AudioDirParms{"samplingrate"}, $AudioDirParms{"samplingrateformat"}, $AudioDirParms{"filetype"}]; 93 } 72 ValidateFilenames($parms, \%AudioDirParms) || confess "UserSubmission error: $?" ; 73 ($AudioDirParms{"samplingrate"}, $AudioDirParms{"samplingrateformat"}, $AudioDirParms{"filetype"}, $AudioDirParms{"dialect"}) 74 = UserSubmission::README::Process($parms, \%AudioDirParms); # can't put "|| confess "UserSubmission error: $?"" becuase changes this to scalar context, and only last item in list is transferred! 75 print "!!!!$audio_dir_name:UserSubmission-README says:".$AudioDirParms{"samplingrate"}."-".$AudioDirParms{"samplingrateformat"}."-".$AudioDirParms{"filetype"}.";dialect:".$AudioDirParms{"dialect"}."\n\n" if $debug; 76 UserSubmission::AUDIO::AudioValidation($parms, \%AudioDirParms) || confess "UserSubmission error: $?" ; #checks that audio rates in README match actual files 77 if (lc $AudioDirParms{"filetype"} eq "flac") { 78 UserSubmission::AUDIO::Flac2Wav($parms, $audio_dir_name); 79 } 80 UserSubmission::PROMPTS::Clean($parms, \%AudioDirParms) || confess "UserSubmission error: $?"; # creates reformatted prompts file for processing with HVite and later merging into Master_Prompts 81 UserSubmission::PROMPTS::Validation($parms, \%AudioDirParms) || confess "UserSubmission error: $?"; # use HDMan to run prompts against VoxForge Dictionnary 82 UserSubmission::AUDIO::Audio2PromptsValidation($parms, \%AudioDirParms) || confess "UserSubmission error: $?" ; # partial AM training session using audio data to see if HVite has any problems trying to re-align the data - if it does, then audio does not match transcription 83 $directorynames{$audio_dir_name} = [$AudioDirParms{"samplingrate"}, $AudioDirParms{"samplingrateformat"}, $AudioDirParms{"filetype"} ]; 84 } 85 94 86 # Move Audio Directories to ToBeProcessed directory 95 87 foreach my $audio_dir_name (sort keys(%directorynames)) { 96 88 my $pointer = $directorynames{$audio_dir_name}; 97 89 my($samplingrate, $samplingrateformat, $filetype) = @$pointer; 98 Copy2ToBeProcessed($parms, $audio_dir_name) || die "UserSubmission error: $?" ; #commit audio to Trunk/Original 90 print "!!!!!!!!!!!!!!!!!!audio_dir_name:$audio_dir_name,filetype:$filetype;\n"; 91 Copy2ToBeProcessed($parms, $audio_dir_name, $filetype) || confess "UserSubmission error: $?" ; #commit audio to Trunk/Original 99 92 } 100 93 # Print Summary … … 114 107 my $audio_dir_name = $$AudioDirParms{"audio_dir_name"}; 115 108 116 opendir(DIR, "$QuarantineDir/$audio_dir_name") || die("Unable to open directory: $QuarantineDir/$audio_dir_name\n");109 opendir(DIR, "$QuarantineDir/$audio_dir_name") || confess ("Unable to open directory: $QuarantineDir/$audio_dir_name\n"); 117 110 while (my $filename = readdir(DIR)) { 118 111 chomp ($filename); … … 121 114 my $suffix = pop(@filename); 122 115 if (defined($suffix)) {} else {$suffix = "";} 123 if ( $suffix ne "wav") {116 if (($suffix ne "wav") and ($suffix ne "flac") and ($suffix ne "raw")) { 124 117 if ($debug) { 125 $command = ("dos2unix -c Mac $QuarantineDir/$audio_dir_name/$filename"); print "$command\n"; system($command) == 0 or die"UserSubmission $command failed: $?";126 $command = ("dos2unix $QuarantineDir/$audio_dir_name/$filename"); print "$command\n"; system($command) == 0 or die"UserSubmission $command failed: $?";118 $command = ("dos2unix -c Mac $QuarantineDir/$audio_dir_name/$filename"); print "$command\n"; system($command) == 0 or confess "UserSubmission $command failed: $?"; 119 $command = ("dos2unix $QuarantineDir/$audio_dir_name/$filename"); print "$command\n"; system($command) == 0 or confess "UserSubmission $command failed: $?"; 127 120 } else { 128 $command = ("dos2unix -c Mac $QuarantineDir/$audio_dir_name/$filename 1>/dev/null 2>/dev/null"); system($command) == 0 or die"UserSubmission $command failed: $?";129 $command = ("dos2unix $QuarantineDir/$audio_dir_name/$filename 1>/dev/null 2>/dev/null"); system($command) == 0 or die"UserSubmission $command failed: $?";121 $command = ("dos2unix -c Mac $QuarantineDir/$audio_dir_name/$filename 1>/dev/null 2>/dev/null"); system($command) == 0 or confess "UserSubmission $command failed: $?"; 122 $command = ("dos2unix $QuarantineDir/$audio_dir_name/$filename 1>/dev/null 2>/dev/null"); system($command) == 0 or confess "UserSubmission $command failed: $?"; 130 123 } 131 124 } … … 138 131 my($Dir) = @_; 139 132 my @tarlist; 140 opendir(DIR, $Dir) || die("Unable to open file: $Dir");133 opendir(DIR, $Dir) || confess ("Unable to open file: $Dir"); 141 134 while (my $line = readdir(DIR)) { 142 135 chomp ($line); 143 136 if ($line ne "." and $line ne ".." ) { 144 # !!!!!!145 137 my @suffixlist = ("tgz", "tar\.gz", "zip"); 146 138 my ($filename,$path,$suffix) = fileparse($line,@suffixlist); 147 148 #my($filename, $suffix1, $suffix2) =split(/\./, $line);149 #if (defined($suffix1)) {} else {$suffix1=""};150 #if (defined($suffix2)) {} else {$suffix2=""};151 # !!!!!!152 153 139 if (($suffix eq "zip") or ($suffix eq "tgz") or ($suffix eq "tar\.gz") ) { 154 140 push @tarlist,$line; … … 172 158 my $filename = $$AudioDirParms {"audio_dir_name"}; 173 159 my $suffix = $$AudioDirParms {"suffix"}; 174 175 160 176 161 my (@clamscan_output, $clamscan_results); 177 #$command = ("freshclam"); print "$command\n"; system($command) == 0 or die"system $command failed: $?";162 #$command = ("freshclam"); print "$command\n"; system($command) == 0 or confess "system $command failed: $?"; 178 163 print "\nRunning ClamAV and Unzipping/Untarring $tarfile\n"; 179 164 # !!!!!! 180 165 print "\nfilename:$filename; suffix:$suffix\n" if $debug; 181 166 if ($suffix eq "zip" ) { 182 $command = ("clamscan --unzip $QuarantineDir/$tarfile"); print "$command\n"; @clamscan_output = `$command`; if ($?) { die"system $command failed: $?"; };167 $command = ("clamscan --unzip $QuarantineDir/$tarfile"); print "$command\n"; @clamscan_output = `$command`; if ($?) {confess "system $command failed: $?"; }; 183 168 my @temp = split (" ", $clamscan_output[0]); 184 169 $clamscan_results = pop (@temp); … … 187 172 print "file $QuarantineDir/$tarfile is clean!\n";} 188 173 else { 189 die"ClamScan found a virus in file: $QuarantineDir\/$tarfile. $?"174 confess "ClamScan found a virus in file: $QuarantineDir\/$tarfile. $?" 190 175 } 191 176 if ($debug) { 192 $command = ("cd $QuarantineDir && unzip -o -j $tarfile -d $filename"); print "$command\n"; system($command) == 0 or die"system $command failed: $?";177 $command = ("cd $QuarantineDir && unzip -o -j $tarfile -d $filename"); print "$command\n"; system($command) == 0 or confess "system $command failed: $?"; 193 178 } else { 194 $command = ("cd $QuarantineDir && unzip -o -j $tarfile -d $filename 1>/dev/null 2>/dev/null");; system($command) == 0 or die"system $command failed: $?";179 $command = ("cd $QuarantineDir && unzip -o -j $tarfile -d $filename 1>/dev/null 2>/dev/null");; system($command) == 0 or confess "system $command failed: $?"; 195 180 } 196 181 197 182 } elsif (($suffix eq "tgz") or ($suffix eq "tar\.gz")) { 198 #$command = ("clamscan --tgz $QuarantineDir/$tarfile"); print "$command\n"; system($command) == 0 or die"system $command failed: $?";199 $command = ("clamscan --tgz $QuarantineDir/$tarfile"); print "$command\n" if $debug; @clamscan_output = `$command`; if ($?) { die"system $command failed: $?"; };183 #$command = ("clamscan --tgz $QuarantineDir/$tarfile"); print "$command\n"; system($command) == 0 or confess "system $command failed: $?"; 184 $command = ("clamscan --tgz $QuarantineDir/$tarfile"); print "$command\n" if $debug; @clamscan_output = `$command`; if ($?) {confess "system $command failed: $?"; }; 200 185 $clamscan_results = "error"; 201 186 foreach my $line (@clamscan_output) { … … 207 192 print "file $QuarantineDir\/$tarfile is clean!\n";} 208 193 else { 209 die"ClamScan found a virus in file: $QuarantineDir\/$tarfile. $?"194 confess "ClamScan found a virus in file: $QuarantineDir\/$tarfile. $?" 210 195 } 211 196 … … 214 199 if ($debug) { 215 200 $command = ("mkdir $QuarantineDir/temp"); print "$command\n"; print "$command\n";system($command); 216 $command = ("cd $QuarantineDir/temp && tar -xvzf \.\./$tarfile "); print "$command\n"; system($command) == 0 or die"system $command failed: $?";217 $command = ("cd $QuarantineDir/ && zip -r temp.zip temp"); print "$command\n"; print "$command\n";system($command) == 0 or die"system $command failed: $?";218 $command = ("cd $QuarantineDir/ && rm -rf temp"); print "$command\n"; print "$command\n";system($command) == 0 or die"system $command failed: $?";219 $command = ("cd $QuarantineDir && unzip -o -j temp.zip -d $filename" ); print "$command\n";system($command) == 0 or die"system $command failed: $?";220 $command = ("cd $QuarantineDir/ && rm -f temp.zip"); print "$command\n"; print "$command\n";system($command) == 0 or die"system $command failed: $?";201 $command = ("cd $QuarantineDir/temp && tar -xvzf \.\./$tarfile "); print "$command\n"; system($command) == 0 or confess "system $command failed: $?"; 202 $command = ("cd $QuarantineDir/ && zip -r temp.zip temp"); print "$command\n"; print "$command\n";system($command) == 0 or confess "system $command failed: $?"; 203 $command = ("cd $QuarantineDir/ && rm -rf temp"); print "$command\n"; print "$command\n";system($command) == 0 or confess "system $command failed: $?"; 204 $command = ("cd $QuarantineDir && unzip -o -j temp.zip -d $filename" ); print "$command\n";system($command) == 0 or confess "system $command failed: $?"; 205 $command = ("cd $QuarantineDir/ && rm -f temp.zip"); print "$command\n"; print "$command\n";system($command) == 0 or confess "system $command failed: $?"; 221 206 } else { 222 207 $command = ("mkdir $QuarantineDir/temp"); system($command); 223 $command = ("cd $QuarantineDir/temp && tar -xvzf \.\./$tarfile 1>/dev/null 2>/dev/null"); system($command) == 0 or die"system $command failed: $?";224 $command = ("cd $QuarantineDir/ && zip -r temp.zip temp 1>/dev/null 2>/dev/null"); system($command) == 0 or die"system $command failed: $?";225 $command = ("cd $QuarantineDir/ && rm -rf temp"); system($command) == 0 or die"system $command failed: $?";226 $command = ("cd $QuarantineDir && unzip -o -j temp.zip -d $filename 1>/dev/null 2>/dev/null" ); system($command) == 0 or die"system $command failed: $?";227 $command = ("cd $QuarantineDir/ && rm -f temp.zip"); system($command) == 0 or die"system $command failed: $?";208 $command = ("cd $QuarantineDir/temp && tar -xvzf \.\./$tarfile 1>/dev/null 2>/dev/null"); system($command) == 0 or confess "system $command failed: $?"; 209 $command = ("cd $QuarantineDir/ && zip -r temp.zip temp 1>/dev/null 2>/dev/null"); system($command) == 0 or confess "system $command failed: $?"; 210 $command = ("cd $QuarantineDir/ && rm -rf temp"); system($command) == 0 or confess "system $command failed: $?"; 211 $command = ("cd $QuarantineDir && unzip -o -j temp.zip -d $filename 1>/dev/null 2>/dev/null" ); system($command) == 0 or confess "system $command failed: $?"; 212 $command = ("cd $QuarantineDir/ && rm -f temp.zip"); system($command) == 0 or confess "system $command failed: $?"; 228 213 } 229 214 } else { 230 die"runClamAV ERROR: $filename wrong suffix type - only \.zip and \.tgz supported!\n";215 confess "runClamAV ERROR: $filename wrong suffix type - only \.zip and \.tgz supported!\n"; 231 216 } 232 217 return 1; … … 239 224 # windows defaults to prompts.txt 240 225 # look into converting to XML and run through XML schema validation ... 241 UserSubmission::README::ValidateFileName($parms, $AudioDirectoryParms) || die"UserSubmission error: $?" ;242 UserSubmission::LICENSE::ValidateFileName($parms, $AudioDirectoryParms) || die"UserSubmission error: $?" ;243 UserSubmission::PROMPTS::ValidateFileName($parms, $AudioDirectoryParms) || die"UserSubmission error: $?" ;226 UserSubmission::README::ValidateFileName($parms, $AudioDirectoryParms) || confess "UserSubmission error: $?" ; 227 UserSubmission::LICENSE::ValidateFileName($parms, $AudioDirectoryParms) || confess "UserSubmission error: $?" ; 228 UserSubmission::PROMPTS::ValidateFileName($parms, $AudioDirectoryParms) || confess "UserSubmission error: $?" ; 244 229 } 245 230 246 231 sub Copy2ToBeProcessed { 247 my ($parms, $audio_dir_name ) = @_;232 my ($parms, $audio_dir_name, $filetype) = @_; 248 233 my $debug = $$parms{"debug"}; 249 234 my $QuarantineDir = $$parms{"QuarantineDir"}; 250 235 my $ToBeProcessedDir = $$parms{"ToBeProcessedDir"}; 251 $command = ("mkdir $QuarantineDir/$audio_dir_name/wav"); print "$command\n" if $debug; system($command); 252 $command = ("mv -f $QuarantineDir/$audio_dir_name/*.wav $QuarantineDir/$audio_dir_name/wav"); print "$command\n" if $debug; system($command) == 0 or die "system $command failed: $?"; 253 $command = ("rm -f $QuarantineDir/$audio_dir_name/*~ $QuarantineDir/$audio_dir_name/*.*~"); print "$command\n" if $debug; system($command) == 0 or die "system $command failed: $?"; # remove hidden backup files 254 # copy to ToBeProcessed directory 236 UserSubmission::AUDIO::Finalize($parms, $audio_dir_name, $filetype) || confess "UserSubmission error: $?" ; 255 237 $command = ("rm -rf $ToBeProcessedDir/$audio_dir_name"); print "$command\n" if $debug; system($command) ; 256 $command = ("mkdir $ToBeProcessedDir/$audio_dir_name $ToBeProcessedDir/$audio_dir_name/wav$ToBeProcessedDir/$audio_dir_name/etc"); print "$command\n" if $debug; system($command) ;257 $command = ("cp -f $QuarantineDir/$audio_dir_name/wav/*.wav $ToBeProcessedDir/$audio_dir_name/wav"); print "$command\n" if $debug; system($command) == 0 or die "system $command failed: $?"; 238 $command = ("mkdir $ToBeProcessedDir/$audio_dir_name $ToBeProcessedDir/$audio_dir_name/etc"); print "$command\n" if $debug; system($command) ; 239 UserSubmission::AUDIO::Copy2ToBeProcessed($parms, $audio_dir_name, $filetype) || confess "UserSubmission error: $?" ; 258 240 if ($debug) { 259 241 $command = ("cp -f $QuarantineDir/$audio_dir_name/* $ToBeProcessedDir/$audio_dir_name/etc"); print "$command\n"; system($command) ; # return value usually says "cp: omitting directory `/data/quarantine/granthulbert-ar-01032007/wav'" … … 261 243 $command = ("cp -f $QuarantineDir/$audio_dir_name/* $ToBeProcessedDir/$audio_dir_name/etc 1>/dev/null 2>/dev/null"); system($command) ; # return value usually says "cp: omitting directory `/data/quarantine/granthulbert-ar-01032007/wav'" 262 244 } 263 UserSubmission::LICENSE::AddGPLLicense($parms, $audio_dir_name) || die"UserSubmission error: $?" ;245 UserSubmission::LICENSE::AddGPLLicense($parms, $audio_dir_name) || confess "UserSubmission error: $?" ; 264 246 if ($debug) { 265 247 $command = ("mv -f $ToBeProcessedDir/$audio_dir_name/etc/LICENSE $ToBeProcessedDir/$audio_dir_name"); print "$command\n" ;system($command) ; … … 268 250 } 269 251 270 $command = ("chown -R kmaclean $ToBeProcessedDir/$audio_dir_name"); print "$command\n" if $debug; system($command) == 0 or die"system $command failed: $?";252 $command = ("chown -R kmaclean $ToBeProcessedDir/$audio_dir_name"); print "$command\n" if $debug; system($command) == 0 or confess "system $command failed: $?"; 271 253 return 1; 272 254 } … … 280 262 my $audio_dir_name = $$AudioDirParms{"audio_dir_name"}; 281 263 282 $command = ("rm -rf $QuarantineDir/$audio_dir_name"); print "$command\n" if $debug; system($command) == 0 or die"system $command failed: $?"; # remove hidden backup files283 $command = ("rm -f $QuarantineDir/$audio_dir_name\.zip $QuarantineDir/$audio_dir_name\.tgz $QuarantineDir/$audio_dir_name\.tar.gz "); print "$command\n" if $debug; system($command) == 0 or die"system $command failed: $?"; # remove hidden backup files284 $command = ("rm -rf $ToBeProcessedDir/$audio_dir_name"); print "$command\n" if $debug; system($command) == 0 or die"system $command failed: $?"; # remove hidden backup files264 $command = ("rm -rf $QuarantineDir/$audio_dir_name"); print "$command\n" if $debug; system($command) == 0 or confess "system $command failed: $?"; # remove hidden backup files 265 $command = ("rm -f $QuarantineDir/$audio_dir_name\.zip $QuarantineDir/$audio_dir_name\.tgz $QuarantineDir/$audio_dir_name\.tar.gz "); print "$command\n" if $debug; system($command) == 0 or confess "system $command failed: $?"; # remove hidden backup files 266 $command = ("rm -rf $ToBeProcessedDir/$audio_dir_name"); print "$command\n" if $debug; system($command) == 0 or confess "system $command failed: $?"; # remove hidden backup files