From 73157ae3d38e1010d13c5374d0fea3e249b432da Mon Sep 17 00:00:00 2001 From: David Roazen Date: Fri, 2 Nov 2012 12:02:59 -0400 Subject: [PATCH 1/3] Allow each pipeline test the max of 10 hours to run The runtime of these tests is extremely variable -- sometimes they will complete almost instantly, other times they will wait in an LSF queue for 5-10+ hours. Minimize timeout errors by setting the timeout for these tests to the maximum of 10 hours. --- .../pipeline/DataProcessingPipelineTest.scala | 4 ++-- .../PacbioProcessingPipelineTest.scala | 2 +- .../examples/DevNullOutputPipelineTest.scala | 2 +- .../ExampleCountLociPipelineTest.scala | 2 +- .../ExampleCountReadsPipelineTest.scala | 2 +- .../ExampleReadFilterPipelineTest.scala | 2 +- .../ExampleRetryMemoryLimitPipelineTest.scala | 2 +- .../ExampleUnifiedGenotyperPipelineTest.scala | 8 ++++---- .../examples/HelloWorldPipelineTest.scala | 20 +++++++++---------- 9 files changed, 22 insertions(+), 22 deletions(-) diff --git a/public/scala/test/org/broadinstitute/sting/queue/pipeline/DataProcessingPipelineTest.scala b/public/scala/test/org/broadinstitute/sting/queue/pipeline/DataProcessingPipelineTest.scala index 944ef7977..60c9d9a59 100644 --- a/public/scala/test/org/broadinstitute/sting/queue/pipeline/DataProcessingPipelineTest.scala +++ b/public/scala/test/org/broadinstitute/sting/queue/pipeline/DataProcessingPipelineTest.scala @@ -28,7 +28,7 @@ import org.testng.annotations.Test import org.broadinstitute.sting.BaseTest class DataProcessingPipelineTest { - @Test + @Test(timeOut=36000000) def testSimpleBAM { val projectName = "test1" val testOut = projectName + ".exampleBAM.bam.clean.dedup.recal.bam" @@ -45,7 +45,7 @@ class DataProcessingPipelineTest { PipelineTest.executeTest(spec) } - @Test + @Test(timeOut=36000000) def testBWAPEBAM { val projectName = "test2" val testOut = projectName + ".exampleBAM.bam.clean.dedup.recal.bam" diff --git a/public/scala/test/org/broadinstitute/sting/queue/pipeline/PacbioProcessingPipelineTest.scala b/public/scala/test/org/broadinstitute/sting/queue/pipeline/PacbioProcessingPipelineTest.scala index 3e9af3e68..dd07cbfdc 100644 --- a/public/scala/test/org/broadinstitute/sting/queue/pipeline/PacbioProcessingPipelineTest.scala +++ b/public/scala/test/org/broadinstitute/sting/queue/pipeline/PacbioProcessingPipelineTest.scala @@ -28,7 +28,7 @@ import org.testng.annotations.Test import org.broadinstitute.sting.BaseTest class PacbioProcessingPipelineTest { - @Test + @Test(timeOut=36000000) def testPacbioProcessingPipeline { val testOut = "exampleBAM.recal.bam" val spec = new PipelineTestSpec diff --git a/public/scala/test/org/broadinstitute/sting/queue/pipeline/examples/DevNullOutputPipelineTest.scala b/public/scala/test/org/broadinstitute/sting/queue/pipeline/examples/DevNullOutputPipelineTest.scala index 92c40acb1..6bc6b56db 100644 --- a/public/scala/test/org/broadinstitute/sting/queue/pipeline/examples/DevNullOutputPipelineTest.scala +++ b/public/scala/test/org/broadinstitute/sting/queue/pipeline/examples/DevNullOutputPipelineTest.scala @@ -53,7 +53,7 @@ import org.broadinstitute.sting.queue.pipeline.{PipelineTest, PipelineTestSpec} import org.broadinstitute.sting.BaseTest class DevNullOutputPipelineTest { - @Test + @Test(timeOut=36000000) def testDevNullOutput() { val spec = new PipelineTestSpec spec.name = "devnulloutput" diff --git a/public/scala/test/org/broadinstitute/sting/queue/pipeline/examples/ExampleCountLociPipelineTest.scala b/public/scala/test/org/broadinstitute/sting/queue/pipeline/examples/ExampleCountLociPipelineTest.scala index 9d885dda2..f52632a7f 100644 --- a/public/scala/test/org/broadinstitute/sting/queue/pipeline/examples/ExampleCountLociPipelineTest.scala +++ b/public/scala/test/org/broadinstitute/sting/queue/pipeline/examples/ExampleCountLociPipelineTest.scala @@ -29,7 +29,7 @@ import org.broadinstitute.sting.queue.pipeline.{PipelineTest, PipelineTestSpec} import org.broadinstitute.sting.BaseTest class ExampleCountLociPipelineTest { - @Test + @Test(timeOut=36000000) def testCountLoci() { val testOut = "count.out" val spec = new PipelineTestSpec diff --git a/public/scala/test/org/broadinstitute/sting/queue/pipeline/examples/ExampleCountReadsPipelineTest.scala b/public/scala/test/org/broadinstitute/sting/queue/pipeline/examples/ExampleCountReadsPipelineTest.scala index 1b965d8d2..c23c12719 100644 --- a/public/scala/test/org/broadinstitute/sting/queue/pipeline/examples/ExampleCountReadsPipelineTest.scala +++ b/public/scala/test/org/broadinstitute/sting/queue/pipeline/examples/ExampleCountReadsPipelineTest.scala @@ -29,7 +29,7 @@ import org.broadinstitute.sting.queue.pipeline.{PipelineTest, PipelineTestSpec} import org.broadinstitute.sting.BaseTest class ExampleCountReadsPipelineTest { - @Test + @Test(timeOut=36000000) def testCountReads() { val spec = new PipelineTestSpec spec.name = "countreads" diff --git a/public/scala/test/org/broadinstitute/sting/queue/pipeline/examples/ExampleReadFilterPipelineTest.scala b/public/scala/test/org/broadinstitute/sting/queue/pipeline/examples/ExampleReadFilterPipelineTest.scala index c6e4c3507..4ffaf7b5c 100644 --- a/public/scala/test/org/broadinstitute/sting/queue/pipeline/examples/ExampleReadFilterPipelineTest.scala +++ b/public/scala/test/org/broadinstitute/sting/queue/pipeline/examples/ExampleReadFilterPipelineTest.scala @@ -77,7 +77,7 @@ import org.broadinstitute.sting.queue.pipeline.{PipelineTest, PipelineTestSpec} import org.broadinstitute.sting.BaseTest class ExampleReadFilterPipelineTest { - @Test + @Test(timeOut=36000000) def testExampleReadFilter() { val spec = new PipelineTestSpec spec.name = "examplereadfilter" diff --git a/public/scala/test/org/broadinstitute/sting/queue/pipeline/examples/ExampleRetryMemoryLimitPipelineTest.scala b/public/scala/test/org/broadinstitute/sting/queue/pipeline/examples/ExampleRetryMemoryLimitPipelineTest.scala index a9a5928fc..0215a389c 100644 --- a/public/scala/test/org/broadinstitute/sting/queue/pipeline/examples/ExampleRetryMemoryLimitPipelineTest.scala +++ b/public/scala/test/org/broadinstitute/sting/queue/pipeline/examples/ExampleRetryMemoryLimitPipelineTest.scala @@ -29,7 +29,7 @@ import org.broadinstitute.sting.queue.pipeline.{PipelineTest, PipelineTestSpec} import org.broadinstitute.sting.BaseTest class ExampleRetryMemoryLimitPipelineTest { - @Test + @Test(timeOut=36000000) def testRetryMemoryLimit() { val spec = new PipelineTestSpec spec.name = "RetryMemoryLimit" diff --git a/public/scala/test/org/broadinstitute/sting/queue/pipeline/examples/ExampleUnifiedGenotyperPipelineTest.scala b/public/scala/test/org/broadinstitute/sting/queue/pipeline/examples/ExampleUnifiedGenotyperPipelineTest.scala index f6fcd7c12..67ac68c28 100644 --- a/public/scala/test/org/broadinstitute/sting/queue/pipeline/examples/ExampleUnifiedGenotyperPipelineTest.scala +++ b/public/scala/test/org/broadinstitute/sting/queue/pipeline/examples/ExampleUnifiedGenotyperPipelineTest.scala @@ -29,7 +29,7 @@ import org.broadinstitute.sting.queue.pipeline.{PipelineTest, PipelineTestSpec} import org.broadinstitute.sting.BaseTest class ExampleUnifiedGenotyperPipelineTest { - @Test + @Test(timeOut=36000000) def testUnifiedGenotyper() { val spec = new PipelineTestSpec spec.name = "unifiedgenotyper" @@ -51,7 +51,7 @@ class ExampleUnifiedGenotyperPipelineTest { Array("vcf_intervals", BaseTest.validationDataLocation + "intervalTest.1.vcf") ).asInstanceOf[Array[Array[Object]]] - @Test(dataProvider = "ugIntervals") + @Test(dataProvider = "ugIntervals", timeOut=36000000) def testUnifiedGenotyperWithIntervals(intervalsName: String, intervalsPath: String) { val spec = new PipelineTestSpec spec.name = "unifiedgenotyper_with_" + intervalsName @@ -64,7 +64,7 @@ class ExampleUnifiedGenotyperPipelineTest { PipelineTest.executeTest(spec) } - @Test + @Test(timeOut=36000000) def testUnifiedGenotyperNoGCOpt() { val spec = new PipelineTestSpec spec.name = "unifiedgenotyper_no_gc_opt" @@ -80,7 +80,7 @@ class ExampleUnifiedGenotyperPipelineTest { @DataProvider(name="resMemReqParams") def getResMemReqParam = Array(Array("mem_free"), Array("virtual_free")).asInstanceOf[Array[Array[Object]]] - @Test(dataProvider = "resMemReqParams") + @Test(dataProvider = "resMemReqParams", timeOut=36000000) def testUnifiedGenotyperResMemReqParam(reqParam: String) { val spec = new PipelineTestSpec spec.name = "unifiedgenotyper_" + reqParam diff --git a/public/scala/test/org/broadinstitute/sting/queue/pipeline/examples/HelloWorldPipelineTest.scala b/public/scala/test/org/broadinstitute/sting/queue/pipeline/examples/HelloWorldPipelineTest.scala index a43727ba6..50fc529dd 100644 --- a/public/scala/test/org/broadinstitute/sting/queue/pipeline/examples/HelloWorldPipelineTest.scala +++ b/public/scala/test/org/broadinstitute/sting/queue/pipeline/examples/HelloWorldPipelineTest.scala @@ -28,7 +28,7 @@ import org.testng.annotations.Test import org.broadinstitute.sting.queue.pipeline.{PipelineTest, PipelineTestSpec} class HelloWorldPipelineTest { - @Test + @Test(timeOut=36000000) def testHelloWorld() { val spec = new PipelineTestSpec spec.name = "HelloWorld" @@ -37,7 +37,7 @@ class HelloWorldPipelineTest { PipelineTest.executeTest(spec) } - @Test + @Test(timeOut=36000000) def testHelloWorldWithRunName() { val spec = new PipelineTestSpec spec.name = "HelloWorldWithRunName" @@ -47,7 +47,7 @@ class HelloWorldPipelineTest { PipelineTest.executeTest(spec) } - @Test + @Test(timeOut=36000000) def testHelloWorldWithMemoryLimit() { val spec = new PipelineTestSpec spec.name = "HelloWorldMemoryLimit" @@ -57,7 +57,7 @@ class HelloWorldPipelineTest { PipelineTest.executeTest(spec) } - @Test + @Test(timeOut=36000000) def testHelloWorldWithPriority() { val spec = new PipelineTestSpec spec.name = "HelloWorldWithPriority" @@ -67,7 +67,7 @@ class HelloWorldPipelineTest { PipelineTest.executeTest(spec) } - @Test + @Test(timeOut=36000000) def testHelloWorldWithLsfResource() { val spec = new PipelineTestSpec spec.name = "HelloWorldWithLsfResource" @@ -77,7 +77,7 @@ class HelloWorldPipelineTest { PipelineTest.executeTest(spec) } - @Test + @Test(timeOut=36000000) def testHelloWorldWithLsfResourceAndMemoryLimit() { val spec = new PipelineTestSpec spec.name = "HelloWorldWithLsfResourceAndMemoryLimit" @@ -87,7 +87,7 @@ class HelloWorldPipelineTest { PipelineTest.executeTest(spec) } - @Test + @Test(timeOut=36000000) def testHelloWorldWithLsfEnvironment() { val spec = new PipelineTestSpec spec.name = "HelloWorldWithLsfEnvironment" @@ -97,7 +97,7 @@ class HelloWorldPipelineTest { PipelineTest.executeTest(spec) } - @Test + @Test(timeOut=36000000) def testHelloWorldWithGridEngineResource() { val spec = new PipelineTestSpec spec.name = "HelloWorldWithGridEngineResource" @@ -107,7 +107,7 @@ class HelloWorldPipelineTest { PipelineTest.executeTest(spec) } - @Test + @Test(timeOut=36000000) def testHelloWorldWithGridEngineResourceAndMemoryLimit() { val spec = new PipelineTestSpec spec.name = "HelloWorldWithGridEngineResourceAndMemoryLimit" @@ -117,7 +117,7 @@ class HelloWorldPipelineTest { PipelineTest.executeTest(spec) } - @Test + @Test(timeOut=36000000) def testHelloWorldWithGridEngineEnvironment() { val spec = new PipelineTestSpec spec.name = "HelloWorldWithGridEngineEnvironment" From 0ab4022f2301ffa5b68346835de64a5992356018 Mon Sep 17 00:00:00 2001 From: Mark DePristo Date: Fri, 2 Nov 2012 14:30:20 -0400 Subject: [PATCH 2/3] Final r119 tribble jar --- settings/repository/org.broad/tribble-119.jar | Bin 319935 -> 319935 bytes 1 file changed, 0 insertions(+), 0 deletions(-) diff --git a/settings/repository/org.broad/tribble-119.jar b/settings/repository/org.broad/tribble-119.jar index ab456938aba23540eb8aedbac454841f220c6bba..c74bea398312b68b2f27a60b2061ef2ebf076ff6 100644 GIT binary patch delta 2458 zcmYjRXHZpF5WeT$1*u9^bPS?05ersCQ9#X7U5Rv*ql5`<=ZnzsNGb$kJrVLJ5on z!9Wmn(TOJG#k}amSxKULuD8SgeV?oUHMVTB3fce`uZP&C&p37(H=ClJGrV9h;$8=5 z2{CKpd8VQ!$gv;IpviB=!Mfx{?Z}I>^@Bq1MlxU+_GSou0Q$SyvFwGw@vu);3 zUf}g?D=eTjniDt#G1zZ{1!7##J03q2d`?7r{8}H5Tf#nYf8@qb+zyY_aJzqV9LKn* zc^oTt0LR+h;e*VPQJB@UNS&TKyZYy=>DoO~6LQc>8@qp*p80WMFiQ}uS)wqEoe;*c zlbWtW`>^Sn!>+>+&z`x!N%>~fTB^2YM6nCHmlsD2LbITJI+}0Fw{Wf2*92mgHup~* zr?~gAwJ{gvOxd=-y-Wqc%T&WEMru5sSZb?Y1)B00>-+U>fci8623#=Nt^jPZLfA?J zwUUx!z#f~Z2iYlw*~{+3nZA6_ni*lJ*_s6!<6-h>R*wqGr$Dlot0S19@&Ddiah=A3 zhGO37`D`~1qgbwD@kn1uTf?66!_jbd8T-jDQEWCwqNCUjR8e`lgRR4ZyflW*LdTI9 z=EwUt{KV35N@e$9R)s5~_?=)+3^7+F>x7Po6c)=Bm_CN*&wL^DcbN!6mx){*3jBhFa>Gk}!)4A` zEV^NS?_%*Ie}LyJ#X>$@#(L42m+ILedgEuKIGz&k^DcVHq9Yg9>8e=4pG3?Nn=vG} z=ZkYOcZ5zn{TUvz+Qmj2JRIv2_3?wkS93;3B}fNpq(>=d-FpQ==oRP_()k-G`-e(~ zeDb(q(guEbZ6gh_#*2EBrP26hDGt-6nLHFcLrUUr^vO-~=FjNqDP81;CqB|OPCD6N zvY3SZGj>Wd@b<%<(nB7ph?S0yM$hULshfK|&r20tjUm5D!>H9YX)m%@F6B!>e1)Jy zI>~z`)kx-CgsW{*7{6}SEmd(|5C4!JIN+@k6ZnDWEU(QU_GMub&4llr+You zArk$|Zx9yVpu7n=uonHtav&V>eGc(in+wrsH|9bFqW4W$X7JySseJY(e1~4sJlKR7 zn@3qac@T`Y-!1S%JaLPrd3FoD(4LY{(;Uqom@c1K&(J{-#uIcDf1xAxz6GRGY619Q z{80g{K%7@dYE7zvk#cq+Oh9LMAtk#O!MA897m?jpM8yM(iCkteEWmhwF^#(6HiV*` z(4iLVxeXK1u)ae~qVK>uv|H}LDn#cJ@~ezS${8hq+bp+~kcjT3L{Bb-#pv%TrSV+v z!fLdW?vnrcT^i4)j2lpIfP)|_vI9?E2pMYDqs=%_f^m&trf)F=N^Qi|MESW z%(xP^pdDLD&+@F2;!CTDUse?@gIG;Ng;&!sCDo*WDyJ{r-(Ev~YHEnb_*%jPwN&qM zE#-evN6&Dnj(8i^lk(y9G;e7=@fzPiy8PGx-dMk+ftJhlKGjLPPxBZxQeJ2y=~CVZ zftYXCM7nKjqCRy^um=6k&D8HyGwJ-anbys-h5DXrA^rMW=w3)G&6PKRBig86bQ{&F zYa7tJo#r~yPWg}9iHBR&Ah+~5EJ delta 2458 zcmX|BXHZmE6n^);LtQ|M$m$S8#fSyFC<=r&AQ(}z8jZw&fCY^W1jV4yM9qvM!mRO) z!2-rIqsGB3@D$6^6cwcRz9b5k(a|8*klb@G$q)EWyQh5Tyww-k>5J?v7S5BvTo6nJ zAt5NuVyqY+lr}S6bn#28b;kdFpVI$YCo++OE|er2A+{P5h8?OVOVsm*7FZ$fbaj^y zbH<+`6t!V){iudbd?Q-9_(l3&Twb}s2L!z0K-ku@0or5+K)_+S+8Y!|Hgw155|7R zuq+-JR&S3nv560g^lf+<(wT202>2JMHEeg(M(iD|OWM8I$Ry*q;B2LZhMv0XiCb7`&<6@vlIkBOD!pL)cQQO(^b6ESTY~!`}IvIT}`MdBTRNK z0Ci3TS!PNzDS4)(+ky)6^pnIz_8v-%QJM597s;s{cU49Z7Aa*-?75i`~Q z`=)NsDI{zN#vKbH+mT_#ZaGOdK_PP`dBTK4G2{Z)lRaX|EcC?1l6YiMc@j@np&&0x zBs0-)Adv*K`gK2&Ok9MreLtzd6H$VXkjG|pZftfs8rGd8Ni3u%gY0laOLRFYL8=tD z+5y+adSdH@7JV-{I3$}il zyW<1Bw-2CA3s=jj7UCG?vPx93D%Y(=HBzJ04HqwsWOUDXF7z8L1i@e-S4IJQV5?m7 z6JJ|k=UEFxFN(^G1>y%&6rL>;3oTK|S}nQ{LZK&K^mj(V?WkC9go4p=(ajcx>6gVa z6$)aW*n}>*O)t(t!?1ku*e59Dw22LB6jJ&`V=D}yTXV-2%R$=9M|yPfl=llk5MBuU z6Uq-URdx@N%-G}y9i%l(=v7OD>@kzkL}>(mS&HjaX$EtJPm|Kwjy`%x{%noDzS21+ zJPweqFlvs6NVXHOepZ4s4PQS@kRCEmS(0>k1SVc_QtDzB-!oDfV`I>7k^`X2^@_9; zyI0QZr7(6vP$K=zYNl67Hta4hw@T6M`QR?8f?)8&Kcr4q6iO`Mdq&h;HS{u8y*x0$HKzpaI&@kA_+9-|!9e!X+;%ew;nG@GSA2xP=%Lb4v@sk_* zkIS57dPK?c@Gi+ouKY_fQ_gON;~0*|>>_7O0yBAYESO`aq*!>zCfl|R5**O-A_d%7 zH76O$S^ej~fF0{mx(E?0&#Fr>8)GvsL8!5q5G23+9X#mog*-tAKD2jvqYfq`X6OKy zEH~=F8&RDN%MfF;VGUwSHhhKXo&!q|59Po@#Eu;J8qq5k<{=))<@VNG_!9N$3M@kW zNr5jATNGH0=zo>_)35S(Jy&5p+Lv78Sa^-cjmv{bw5Q}j4C1>y-cRIph(o>MI;=zV zzX6L)|NAkOPu+lTaZ1aZumLgYCXJ%la}&ZbI`|d@BObX$r=fH|y#;=#Ptx;g4(SJ` z)6=f0e_B3-A*SY2KgH+yy!*fc&QeAJ1mOEe1+WzHvqH|+geq{9a|>Y{8oLU4uxAl` zgZl9zuJ;wuAk)OMPz=QfN-#r_WOH0}d$*P|KDLhXLi7Jn2mYA9q>dk#XFac*S39={uSyqJ{H!6Oq0j*2?R}wbDG) zU(-6kcN?9H>Id3*e0Lk~!?T^&p_dYb{-$%SV%Lpr&h*~#-01B77w Id;`S(3mt0_!vFvP From eae2d019cf6f9b5b359a3bbc0b0066f490464f44 Mon Sep 17 00:00:00 2001 From: David Roazen Date: Fri, 2 Nov 2012 19:01:59 -0400 Subject: [PATCH 3/3] Refuse to package the GATK from a non-clean working directory Packaging from a non-clean working directory can result in an incorrect jar. Now that we have external collaborators packaging and distributing the GATK, not enforcing the clean requirement has become far too dangerous. At the same time, invoking "clean" automatically through a direct dependency would also be dangerous -- instead, it's better to error out if a packaging target is invoked from a non-clean working dir. --- build.xml | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/build.xml b/build.xml index c6b1afc56..aa5792419 100644 --- a/build.xml +++ b/build.xml @@ -891,7 +891,7 @@ - + @@ -1011,6 +1011,24 @@ + + + + + + + + + + + + + + + + + +