diff --git a/perl/runPipelineSanityCheck.pl b/perl/runPipelineSanityCheck.pl new file mode 100755 index 000000000..6c2a3c382 --- /dev/null +++ b/perl/runPipelineSanityCheck.pl @@ -0,0 +1,177 @@ +#!/usr/bin/perl -w + +use strict; +use Getopt::Long; +use Data::Dumper; +use File::Path; + +my @log; + +sub getTimestamp { + my ($sec, $min, $hour, $mday, $mon, $year, $wday, $yday, $isdst) = localtime(time); + my $timestamp = sprintf("%04d.%02d.%02d", 1900 + $year, $mon, $mday); + + return $timestamp; +} + +sub log { + my ($message) = @_; + + my ($sec, $min, $hour, $mday, $mon, $year, $wday, $yday, $isdst) = localtime(time); + my $timestamp = sprintf("[%04d.%02d.%02d %02d:%02d:%02d]", 1900 + $year, $mon, $mday, $hour, $min, $sec); + + my @message = split(/\n/, $message); + for (my $i = 0; $i <= $#message; $i++) { + $message[$i] = "$timestamp $message[$i]"; + } + + print STDOUT join("\n", @message) . "\n"; + + push(@log, @message); +} + +sub emailLog { + my ($subject, $logfile, $email) = @_; + + my $cmd = "mail -s '$subject' $email < $logfile"; + system($cmd); +} + +sub killPipeline { + chomp(my @jobs = qx(bjobs -P QueuePipelineTest)); + + my @ids; + foreach my $job (@jobs) { + if ($job =~ /^(\d+)\s+/) { + push(@ids, $1); + } + } + + system("bkill " . join(" ", @ids)); +} + +sub execute { + my ($command, $output, $timeout, $run, $logfile, $email) = @_; + + my $status = 0; + my $result = "(output not generated in dry-run mode)"; + + &log("Execute: '$command'"); + + if ($run == 1) { + eval { + local $SIG{'ALRM'} = sub { die("alarm\n") }; + alarm($timeout); + + $result = qx($command 2>&1); + $status = $?; + + alarm(0); + }; + + if ($@) { + &log("Timeout: '$command' did not return within '$timeout' seconds"); + + if ($run == 1) { + open(LOG, ">$logfile"); + print LOG join("\n", @log); + close(LOG); + + &emailLog("Analysis pipeline nightly test: failed", $logfile, $email); + &killPipeline(); + } + + exit($status); + } + } + + if ($status == 0) { + if ($output == 0) { + &log("Success: '$command' exited with code '$status'"); + } else { + &log("Success: '$command' exited with code '$status' Output:"); + &log($result); + } + + return 0; + } + + &log("Failure: '$command' exited with code '$status' Output:"); + + if (!defined($result)) { + $result = "(unable to capture)\n"; + } + + &log($result); + + if ($run == 1) { + open(LOG, ">$logfile"); + print LOG join("\n", @log); + close(LOG); + + &emailLog("Analysis pipeline nightly test: failed", $logfile, $email); + &killPipeline(); + } + + exit($status); +} + +# Default arguments +my %args = ( + 'pipelineRoot' => "/humgen/gsa-hpprojects/analysis/pipeline", + 'outputRoot' => "/humgen/gsa-hpprojects/analysis/pipeline/projects/QueuePipelineNightlyTest", + 'testRepositoryDir' => "/humgen/gsa-hpprojects/analysis/pipeline/repositories/StingTest", + 'testInputYAML' => "/humgen/gsa-hpprojects/GATK/data/Validation_Data/QueuePipelineTestData/QueuePipelineTestData.yaml", + 'email' => "$ENV{'USER'}\@broadinstitute.org", + 'run' => 0, +); + +# Get command-line options +GetOptions( + 'outputRoot=s' => \$args{'outputRoot'}, + 'pipelineRoot=s' => \$args{'pipelineRoot'}, + 'testRepositoryDir=s' => \$args{'testRepositoryDir'}, + 'testInputYAML=s' => \$args{'testInputYAML'}, + 'email=s' => \$args{'email'}, + 'run' => \$args{'run'}, + 'help|h' => \$args{'help'} +); + +# Should we emit some help? +if ($args{'help'}) { + print Dumper(\%args) . "\n"; + exit(0); +} + +# Set some directories +$args{'testDir'} = "$args{'outputRoot'}/" . &getTimestamp(); +$args{'testOutputDir'} = "$args{'testDir'}/output"; +$args{'logFile'} = "$args{'testDir'}/output.log"; + +# Set up environment +$ENV{'PATH'} = "$args{'testRepositoryDir'}/shell:$args{'testRepositoryDir'}/python:$ENV{'PATH'}"; + +# Execute jobs +&execute("echo \$PATH", 1, 10, $args{'run'}, $args{'logFile'}, $args{'email'}); +&execute("mkdir -p $args{'testDir'}", 0, 10, $args{'run'}, $args{'logFile'}, $args{'email'}); +&execute("mkdir -p $args{'testOutputDir'}", 0, 10, $args{'run'}, $args{'logFile'}, $args{'email'}); +&execute("rm -rf $args{'logFile'}", 0, 10, $args{'run'}, $args{'logFile'}, $args{'email'}); + +&execute("cd $args{'testRepositoryDir'} && svn cleanup", 0, 120, $args{'run'}, $args{'logFile'}, $args{'email'}); +&execute("cd $args{'testRepositoryDir'} && svn up", 0, 600, $args{'run'}, $args{'logFile'}, $args{'email'}); +&execute("cd $args{'testRepositoryDir'} && ant clean", 0, 600, $args{'run'}, $args{'logFile'}, $args{'email'}); +&execute("cd $args{'testRepositoryDir'} && ant dist playground oneoffs queue", 0, 1200, $args{'run'}, $args{'logFile'}, $args{'email'}); +&execute("cd $args{'testRepositoryDir'} && svn info", 1, 10, $args{'run'}, $args{'logFile'}, $args{'email'}); + +&execute("cd $args{'testOutputDir'} && java -jar $args{'testRepositoryDir'}/dist/Queue.jar -jobProject QueuePipelineTest -S $args{'testRepositoryDir'}/scala/qscript/fullCallingPipeline.q -Y $args{'testInputYAML'} -refseqTable /humgen/gsa-hpprojects/GATK/data/Annotations/refseq/refGene-big-table-hg18.txt --gatkjar $args{'testRepositoryDir'}/dist/GenomeAnalysisTK.jar -contigIntervals /humgen/gsa-hpprojects/analysis/pipeline/resources/whole_genome.intervals -titv 3.0 -skipCleaning -bsub -bsubWait", 0, 120, $args{'run'}, $args{'logFile'}, $args{'email'}); +&execute("cd $args{'testOutputDir'} && java -jar $args{'testRepositoryDir'}/dist/Queue.jar -jobProject QueuePipelineTest -S $args{'testRepositoryDir'}/scala/qscript/fullCallingPipeline.q -Y $args{'testInputYAML'} -refseqTable /humgen/gsa-hpprojects/GATK/data/Annotations/refseq/refGene-big-table-hg18.txt --gatkjar $args{'testRepositoryDir'}/dist/GenomeAnalysisTK.jar -contigIntervals /humgen/gsa-hpprojects/analysis/pipeline/resources/whole_genome.intervals -titv 3.0 -skipCleaning -bsub -bsubWait -run", 0, 1200, $args{'run'}, $args{'logFile'}, $args{'email'}); + +&log("All tests completed successfully"); + +if ($args{'run'} == 1) { + open(LOG, ">$args{'logFile'}"); + print LOG join("\n", @log); + close(LOG); + + &emailLog("Analysis pipeline nightly test: succeeded", $args{'logFile'}, $args{'email'}); +}