zzh
/
gatk-3.8
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Command lines for locally run Queue jobs no longer have to be escaped differently than bsub'ed jobs. 

GSA-410 Local job runs now can run command lines longer than than 4096 on our linux machines.
When determining if the help text and Queue extensions need to be rebuilt, use the .class files not the .java so that GATK oneoffs are picked up correctly.
Added the most basic of all example QScripts for debugging, Hello World.
Minor updates to copy/pasted LSF code to reduce ant javadoc warnings by a third.


git-svn-id: file:///humgen/gsa-scr1/gsa-engineering/svn_contents/trunk@4970 348d0f76-0448-11de-a6fe-93d51630548a
This commit is contained in:
kshakir 2011-01-10 21:07:29 +00:00
parent ee348ac9d4
commit 8ba3a5a43f
12 changed files with 310 additions and 200 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
build.xml
View File

@ -48,6 +48,10 @@
<patternset refid="java.source.pattern" />
</fileset>
<fileset id="java.class.files" dir="${java.classes}" erroronmissingdir="false">
<include name="**/*.class"/>
</fileset>
<!-- the path for resources that need to go into the GATK jar;
any additional resources should go into this set. -->
<path id="gatk.resources">
@ -227,8 +231,7 @@
<isset property="scala.include"/>
</not>
<uptodate targetfile="${queue-extensions.source.dir}">
<srcfiles refid="java.source.files"/>
<srcfiles refid="external.source.files"/>
<srcfiles refid="java.class.files"/>
</uptodate>
</or>
</condition>
@ -265,8 +268,7 @@
<or>
<isset property="disable.help"/>
<uptodate targetfile="${basedir}/${resource.path}">
<srcfiles refid="java.source.files"/>
<srcfiles refid="external.source.files"/>
<srcfiles refid="java.class.files"/>
</uptodate>
</or>
</condition>

304
java/src/org/broadinstitute/sting/jna/lsf/v7_0_6/LibBat.java
View File

@ -15525,7 +15525,7 @@ public static class ByValue extends jobArrayElementLog implements Structure.ByVa
* <b>Define Statements:</b>
* \par
* \ref _consumertype
* @see \ref lsb_freeLimitInfoEnt
* #see \ref lsb_freeLimitInfoEnt
*/
public static native int lsb_limitInfo(_limitInfoReq req, Pointer limitItemRef, IntByReference size, LibLsf.lsInfo lsInfo);
@ -15567,7 +15567,7 @@ public static class ByValue extends jobArrayElementLog implements Structure.ByVa
* $LSB_CONFDIR/cluster_name/configdir/lsb.hosts \n
* $LSB_CONFDIR/cluster_name/configdir/lsb.resources
* @param ent input, the array of limit information
* @see \ref lsb_limitInfo
* #see \ref lsb_limitInfo
*/
public static native void lsb_freeLimitInfoEnt(_limitInfoEnt ent, int size);
@ -16023,8 +16023,8 @@ public static class ByValue extends jobArrayElementLog implements Structure.ByVa
* <b>Define Statements:</b>
* \par
* none
* @see \ref lsb_usergrpinfo
* @see \ref lsb_hostgrpinfo
* #see \ref lsb_usergrpinfo
* #see \ref lsb_hostgrpinfo
* @param stringArray1 stringArray1
*/
public static native hostPartInfoEnt.ByReference lsb_hostpartinfo(Pointer stringArray1, IntByReference numHostHosts);
@ -16135,12 +16135,12 @@ public static class ByValue extends jobArrayElementLog implements Structure.ByVa
* @param hostName Passes information about jobs on the named host, host
* group or cluster name. If host is null, jobs on all hosts of the batch
* system will be considered.
* @see \ref lsb_openjobinfo_a
* @see \ref lsb_openjobinfo_a_ext
* @see \ref lsb_openjobinfo_req
* @see \ref lsb_closejobinfo
* @see \ref lsb_readjobinfo
* @see \ref lsb_readframejob
* #see \ref lsb_openjobinfo_a
* #see \ref lsb_openjobinfo_a_ext
* #see \ref lsb_openjobinfo_req
* #see \ref lsb_closejobinfo
* #see \ref lsb_readjobinfo
* #see \ref lsb_readframejob
*/
public static native int lsb_openjobinfo(long jobId, String jobName, String userName, String queueName, String hostName, int options);
@ -16209,10 +16209,10 @@ public static class ByValue extends jobArrayElementLog implements Structure.ByVa
* @param hostName Passes information about jobs on the named host, host group
* or cluster name. If hostName is null, jobs on all hosts of the batch system
* will be considered.
* @see \ref lsb_openjobinfo
* @see \ref lsb_closejobinfo
* @see \ref lsb_readjobinfo
* @see \ref lsb_readframejob
* #see \ref lsb_openjobinfo
* #see \ref lsb_closejobinfo
* #see \ref lsb_readjobinfo
* #see \ref lsb_readframejob
*/
public static native jobInfoHead.ByReference lsb_openjobinfo_a(long jobId, String jobName, String userName, String queueName, String hostName, int options);
@ -16278,10 +16278,10 @@ public static class ByValue extends jobArrayElementLog implements Structure.ByVa
* @param hostName Passes information about jobs on the named host, host group
* or cluster name. If hostName is null, jobs on all hosts of the batch system
* will be considered.
* @see \ref lsb_openjobinfo
* @see \ref lsb_closejobinfo
* @see \ref lsb_readjobinfo
* @see \ref lsb_readframejob
* #see \ref lsb_openjobinfo
* #see \ref lsb_closejobinfo
* #see \ref lsb_readjobinfo
* #see \ref lsb_readframejob
*/
public static native jobInfoHeadExt.ByReference lsb_openjobinfo_a_ext(long jobId, String jobName, String userName, String queueName, String hostName, int options);
@ -16322,11 +16322,11 @@ public static class ByValue extends jobArrayElementLog implements Structure.ByVa
* \par
* \ref defs_lsb_openjobinfo_a
* \n \ref defs_lsb_openjobinfo
* @see \ref lsb_openjobinfo_a
* @see \ref lsb_openjobinfo_a_ext
* @see \ref lsb_closejobinfo
* @see \ref lsb_readjobinfo
* @see \ref lsb_readframejob
* #see \ref lsb_openjobinfo_a
* #see \ref lsb_openjobinfo_a_ext
* #see \ref lsb_closejobinfo
* #see \ref lsb_readjobinfo
* #see \ref lsb_readframejob
*/
public static native jobInfoHeadExt.ByReference lsb_openjobinfo_req(jobInfoReq req);
@ -16382,13 +16382,13 @@ public static class ByValue extends jobArrayElementLog implements Structure.ByVa
* \ref job_states
* \n \ref jobgroup_counterIndex
* \n \ref group_nodetypes
* @see \ref lsb_openjobinfo
* @see \ref lsb_openjobinfo_a
* @see \ref lsb_closejobinfo
* @see \ref lsb_hostinfo
* @see \ref lsb_pendreason
* @see \ref lsb_queueinfo
* @see \ref lsb_suspreason
* #see \ref lsb_openjobinfo
* #see \ref lsb_openjobinfo_a
* #see \ref lsb_closejobinfo
* #see \ref lsb_hostinfo
* #see \ref lsb_pendreason
* #see \ref lsb_queueinfo
* #see \ref lsb_suspreason
*/
public static native jobInfoEnt.ByReference lsb_readjobinfo(IntByReference more);
@ -16440,9 +16440,9 @@ public static class ByValue extends jobArrayElementLog implements Structure.ByVa
* \ref lsb_submit_options
* \n \ref lsb_submit_options2
* \n \ref lsb_submit_options3
* @see \ref lsb_modify
* @see \ref ls_info
* @see \ref lsb_queueinfo
* #see \ref lsb_modify
* #see \ref ls_info
* #see \ref lsb_queueinfo
*/
public static native long lsb_submit(submit jobSubReq, submitReply jobSubReply);
@ -16499,14 +16499,14 @@ public static class ByValue extends jobArrayElementLog implements Structure.ByVa
* \par
* \ref external_msg_processing
* \n \ref group_nodetypes
* @see \ref lsb_openjobinfo
* @see \ref lsb_openjobinfo_a
* @see \ref lsb_closejobinfo
* @see \ref lsb_hostinfo
* @see \ref lsb_pendreason
* @see \ref lsb_queueinfo
* @see \ref lsb_readjobinfo
* @see \ref lsb_suspreason
* #see \ref lsb_openjobinfo
* #see \ref lsb_openjobinfo_a
* #see \ref lsb_closejobinfo
* #see \ref lsb_hostinfo
* #see \ref lsb_pendreason
* #see \ref lsb_queueinfo
* #see \ref lsb_readjobinfo
* #see \ref lsb_suspreason
*/
public static native jobInfoEnt.ByReference lsb_readjobinfo_cond(IntByReference more, jobInfoHeadExt jInfoHExt);
@ -16566,9 +16566,9 @@ public static class ByValue extends jobArrayElementLog implements Structure.ByVa
* <b>Define Statements:</b>
* \par
* none
* @see \ref lsb_openjobinfo
* @see \ref lsb_readjobinfo
* @see \ref lsb_closejobinfo
* #see \ref lsb_openjobinfo
* #see \ref lsb_readjobinfo
* #see \ref lsb_closejobinfo
*/
public static native int lsb_readframejob(long jobId, String frameName, String user, String queue, String host, int options, Pointer frameJobInfoTbl);
@ -16609,9 +16609,9 @@ public static class ByValue extends jobArrayElementLog implements Structure.ByVa
* <b>Files:</b>
* \par
* ${LSF_ENVDIR:-/etc}/lsf.conf
* @see \ref lsb_openjobinfo
* @see \ref lsb_openjobinfo_a
* @see \ref lsb_readjobinfo
* #see \ref lsb_openjobinfo
* #see \ref lsb_openjobinfo_a
* #see \ref lsb_readjobinfo
*/
public static native void lsb_closejobinfo();
@ -16654,7 +16654,7 @@ public static class ByValue extends jobArrayElementLog implements Structure.ByVa
* <b>Define Statements:</b>
* \par
* \ref host_ctrl_option
* @see \ref lsb_reconfig
* #see \ref lsb_reconfig
*/
public static native int lsb_hostcontrol(hostCtrlReq req);
@ -16713,9 +16713,9 @@ public static class ByValue extends jobArrayElementLog implements Structure.ByVa
* the queues that are enabled for the hosts are of interest.
* @param user The name of user. If user is not null, then only the queues
* that are enabled for the user are of interest.
* @see \ref lsb_hostinfo
* @see \ref lsb_userinfo
* @see \ref lsb_usergrpinfo
* #see \ref lsb_hostinfo
* #see \ref lsb_userinfo
* #see \ref lsb_usergrpinfo
*/
public static native queueInfoEnt.ByReference lsb_queueinfo(Pointer queues, IntByReference numQueues, String hosts, String user, int options);
@ -16760,7 +16760,7 @@ public static class ByValue extends jobArrayElementLog implements Structure.ByVa
* <b>Define Statements:</b>
* \par
* \ref mbd_operation
* @see \ref lsb_openjobinfo
* #see \ref lsb_openjobinfo
*/
public static native int lsb_reconfig(mbdCtrlReq req);
@ -16810,9 +16810,9 @@ public static class ByValue extends jobArrayElementLog implements Structure.ByVa
* <b>Files:</b>
* \par
* ${LSF_ENVDIR:-/etc}/lsf.conf
* @see \ref lsb_chkpntjob
* @see \ref lsb_forcekilljob
* @see \ref lsb_mig
* #see \ref lsb_chkpntjob
* #see \ref lsb_forcekilljob
* #see \ref lsb_mig
*/
public static native int lsb_signaljob(long jobId, int sigValue);
@ -16955,8 +16955,8 @@ public static class ByValue extends jobArrayElementLog implements Structure.ByVa
* <b>Files:</b>
* \par
* ${LSF_ENVDIR:-/etc}/lsf.conf
* @see \ref lsb_signaljob
* @see \ref lsb_chkpntjob
* #see \ref lsb_signaljob
* #see \ref lsb_chkpntjob
*/
public static native int lsb_deletejob(long jobId, int times, int options);
@ -16991,7 +16991,7 @@ public static class ByValue extends jobArrayElementLog implements Structure.ByVa
* <b>Files:</b>
* \par
* none
* @see \ref lsb_signaljob
* #see \ref lsb_signaljob
*/
public static native int lsb_forcekilljob(long jobId);
@ -17124,8 +17124,8 @@ public static class ByValue extends jobArrayElementLog implements Structure.ByVa
* <b>Define Statements:</b>
* \par
* \ref requeuejob_options
* @see \ref lsb_movejob
* @see \ref lsb_pendreason
* #see \ref lsb_movejob
* #see \ref lsb_pendreason
*/
public static native int lsb_requeuejob(jobrequeue reqPtr);
@ -17167,8 +17167,8 @@ public static class ByValue extends jobArrayElementLog implements Structure.ByVa
* <b>Files:</b>
* \par
* none
* @see \ref ls_perror
* @see \ref ls_sysmsg
* #see \ref ls_perror
* #see \ref ls_sysmsg
*/
public static native String lsb_sysmsg();
@ -17304,7 +17304,7 @@ public static class ByValue extends jobArrayElementLog implements Structure.ByVa
* <b>Define Statements:</b>
* \par
* none
* @see \ref lsb_submit
* #see \ref lsb_submit
*/
public static native int lsb_mig(submig mig, IntByReference badHostIdx);
@ -17354,11 +17354,11 @@ public static class ByValue extends jobArrayElementLog implements Structure.ByVa
* \ref host_status
* \n \ref host_load_BusyReason
* \n \ref host_attributes
* @see \ref lsb_hostinfo_ex
* @see \ref ls_info
* @see \ref ls_loadofhosts
* @see \ref lsb_queueinfo
* @see \ref lsb_userinfo
* #see \ref lsb_hostinfo_ex
* #see \ref ls_info
* #see \ref ls_loadofhosts
* #see \ref lsb_queueinfo
* #see \ref lsb_userinfo
*/
public static native hostInfoEnt.ByReference lsb_hostinfo(Pointer hosts, IntByReference numHosts);
@ -17410,11 +17410,11 @@ public static class ByValue extends jobArrayElementLog implements Structure.ByVa
* that satisfy the resource requirements is returned. Returned hosts are
* sorted according to the load on the resource() given in resReq, or by
* default according to CPU and paging load.
* @see \ref ls_info
* @see \ref ls_loadofhosts
* @see \ref lsb_hostinfo
* @see \ref lsb_queueinfo
* @see \ref lsb_userinfo
* #see \ref ls_info
* #see \ref ls_loadofhosts
* #see \ref lsb_hostinfo
* #see \ref lsb_queueinfo
* #see \ref lsb_userinfo
* @param string1 string1
*/
@ -17469,7 +17469,7 @@ public static class ByValue extends jobArrayElementLog implements Structure.ByVa
* * numHosts will be set to the actual number of hostInfoEnt structures in
* the host group when this call returns.
* @param resReq Any resource requirements called with the function.
* @see \ref lsb_hostinfo
* #see \ref lsb_hostinfo
*/
public static native condHostInfoEnt.ByReference lsb_hostinfo_cond(Pointer hosts, IntByReference numHosts, String resReq, int options);
@ -17530,7 +17530,7 @@ public static class ByValue extends jobArrayElementLog implements Structure.ByVa
* ${LSF_ENVDIR:-/etc}/lsf.conf
* @param position The new position of the job in a queue. position must be
* a value of 1 or more.
* @see \ref lsb_pendreason
* #see \ref lsb_pendreason
*/
public static native int lsb_movejob(long jobId, IntByReference opCode, int position);
@ -17623,7 +17623,7 @@ public static class ByValue extends jobArrayElementLog implements Structure.ByVa
* <b>Define Statements:</b>
* \par
* \ref queue_ctrl_option
* @see \ref lsb_queueinfo
* #see \ref lsb_queueinfo
*/
public static native int lsb_queuecontrol(queueCtrlReq req);
@ -17677,8 +17677,8 @@ public static class ByValue extends jobArrayElementLog implements Structure.ByVa
* <b>Define Statements:</b>
* \par
* none
* @see \ref lsb_hostinfo
* @see \ref lsb_queueinfo
* #see \ref lsb_hostinfo
* #see \ref lsb_queueinfo
*/
public static native userInfoEnt.ByReference lsb_userinfo(Pointer users, IntByReference numUsers);
@ -17737,7 +17737,7 @@ public static class ByValue extends jobArrayElementLog implements Structure.ByVa
* @param groups An array of group names.
* @param numGroups The number of group names.* numGroups will be updated
* to the actual number of groups when this call returns.
* @see \ref lsb_usergrpinfo
* #see \ref lsb_usergrpinfo
*/
public static native groupInfoEnt.ByReference lsb_hostgrpinfo(Pointer groups, IntByReference numGroups, int options);
@ -17787,7 +17787,7 @@ public static class ByValue extends jobArrayElementLog implements Structure.ByVa
* @param groups An array of group names.
* @param numGroups The number of group names.* numGroups will be updated
* to the actual number of groups when this call returns.
* @see \ref lsb_hostgrpinfo
* #see \ref lsb_hostgrpinfo
*/
public static native groupInfoEnt.ByReference lsb_usergrpinfo(Pointer groups, IntByReference numGroups, int options);
@ -17876,10 +17876,10 @@ public static class ByValue extends jobArrayElementLog implements Structure.ByVa
* to the batch system and an error is returned.
* @param jobSubReply Describes the results of the job modification to the
* batch system.
* @see \ref lsb_submit
* @see \ref ls_info
* @see \ref ls_rtask
* @see \ref lsb_queueinfo
* #see \ref lsb_submit
* #see \ref ls_info
* #see \ref ls_rtask
* #see \ref lsb_queueinfo
*/
public static native long lsb_modify(submit jobSubReq, submitReply jobSubReply, long jobId);
@ -17964,7 +17964,7 @@ public static class ByValue extends jobArrayElementLog implements Structure.ByVa
* \par
* \ref suspending_reasons \n
* \ref suspending_subreasons
* @see \ref lsb_pendreason
* #see \ref lsb_pendreason
*/
public static native String lsb_suspreason(int reasons, int subreasons, loadIndexLog ld);
@ -18017,7 +18017,7 @@ public static class ByValue extends jobArrayElementLog implements Structure.ByVa
* @param ld From \ref lsb_suspreason, when reasons is SUSP_LOAD_REASON, ld is used to
* determine the name of any external load indices. ld uses the most recent load
* index log in the lsb.events file.
* @see \ref lsb_geteventrec
* #see \ref lsb_geteventrec
*/
public static native String lsb_pendreason(int numReasons, IntByReference rsTb, jobInfoHead jInfoH, loadIndexLog ld, int clusterId);
@ -18065,7 +18065,7 @@ public static class ByValue extends jobArrayElementLog implements Structure.ByVa
* <b>Define Statements:</b>
* \par
* none
* @see \ref lsb_calendarop
* #see \ref lsb_calendarop
*/
public static native calendarInfoEnt.ByReference lsb_calendarinfo(Pointer calendars, IntByReference numCalendars, String user);
@ -18131,7 +18131,7 @@ public static class ByValue extends jobArrayElementLog implements Structure.ByVa
* <b>Define Statements:</b>
* \par
* \ref calendar_command
* @see \ref lsb_calendarinfo
* #see \ref lsb_calendarinfo
*/
public static native int lsb_calendarop(int oper, int numNames, Pointer names, String desc, String calExpr, int options, String badStr);
@ -18238,7 +18238,7 @@ public static class ByValue extends jobArrayElementLog implements Structure.ByVa
* \n \ref defs_lsb_XF_OP
* \n \ref jobgroup_controltypes
* \n \ref signal_action
* @see \ref lsb_geteventrec
* #see \ref lsb_geteventrec
*/
public static native int lsb_puteventrec(Pointer logPtr, eventRec log_fp);
@ -18348,14 +18348,14 @@ public static class ByValue extends jobArrayElementLog implements Structure.ByVa
* \n \ref defs_lsb_XF_OP
* \n \ref jobgroup_controltypes
* \n \ref signal_action
* @see \ref lsb_hostcontrol
* @see \ref lsb_movejob
* @see \ref lsb_pendreason
* @see \ref lsb_puteventrec
* @see \ref lsb_queuecontrol
* @see \ref lsb_readjobinfo
* @see \ref lsb_submit
* @see \ref lsb_suspreason
* #see \ref lsb_hostcontrol
* #see \ref lsb_movejob
* #see \ref lsb_pendreason
* #see \ref lsb_puteventrec
* #see \ref lsb_queuecontrol
* #see \ref lsb_readjobinfo
* #see \ref lsb_submit
* #see \ref lsb_suspreason
*/
public static native eventRec.ByReference lsb_geteventrec(Pointer log_fp, IntByReference lineNum);
@ -18418,7 +18418,7 @@ public static class ByValue extends jobArrayElementLog implements Structure.ByVa
* @param hostName hostName is a string containing a host name. Only shared resource
* available on the specified host will be returned. If hostName is a null,
* shared resource available on all hosts will be returned.
* @see \ref ls_sharedresourceinfo
* #see \ref ls_sharedresourceinfo
*/
public static native Pointer lsb_sharedresourceinfo(Pointer resources, IntByReference numResources, String hostName, int options);
@ -18716,7 +18716,7 @@ public static class ByValue extends jobArrayElementLog implements Structure.ByVa
* \n $JOB_ATTA_DIR
* \n $LSB_SHAREDIR/info
*
* @see \ref lsb_readjobmsg
* #see \ref lsb_readjobmsg
*
*/
@ -18785,7 +18785,7 @@ public static class ByValue extends jobArrayElementLog implements Structure.ByVa
* $LSB_CONFDIR/cluster_name/configdir/lsb.params
* \n $JOB_ATTA_DIR
* \n $LSB_SHAREDIR/info
* @see \ref lsb_postjobmsg
* #see \ref lsb_postjobmsg
*/
public static native int lsb_readjobmsg(jobExternalMsgReq jobExternalMsg, jobExternalMsgReply jobExternalMsgReply);
@ -18836,9 +18836,9 @@ public static class ByValue extends jobArrayElementLog implements Structure.ByVa
* <b>Define Statements:</b>
* \par
* \ref reservation_option
* @see \ref lsb_removereservation
* @see \ref lsb_modreservation
* @see \ref lsb_reservationinfo
* #see \ref lsb_removereservation
* #see \ref lsb_modreservation
* #see \ref lsb_reservationinfo
*/
public static native int lsb_addreservation(addRsvRequest request, String rsvId);
@ -18876,9 +18876,9 @@ public static class ByValue extends jobArrayElementLog implements Structure.ByVa
* <b>Define Statements:</b>
* \par
* none
* @see \ref lsb_addreservation
* @see \ref lsb_modreservation
* @see \ref lsb_reservationinfo
* #see \ref lsb_addreservation
* #see \ref lsb_modreservation
* #see \ref lsb_reservationinfo
*/
public static native int lsb_removereservation(String rsvId);
@ -18931,9 +18931,9 @@ public static class ByValue extends jobArrayElementLog implements Structure.ByVa
* none
* @param rsvId Reservation ID of the requested reservation.
* @param numEnts Number of reservation entries that mbatchd returns.
* @see \ref lsb_addreservation
* @see \ref lsb_modreservation
* @see \ref lsb_removereservation
* #see \ref lsb_addreservation
* #see \ref lsb_modreservation
* #see \ref lsb_removereservation
*/
public static native rsvInfoEnt.ByReference lsb_reservationinfo(String rsvId, IntByReference numEnts, int options);
@ -18981,9 +18981,9 @@ public static class ByValue extends jobArrayElementLog implements Structure.ByVa
* <b>Define Statements:</b>
* \par
* none
* @see \ref lsb_addreservation
* @see \ref lsb_removereservation
* @see \ref lsb_reservationinfo
* #see \ref lsb_addreservation
* #see \ref lsb_removereservation
* #see \ref lsb_reservationinfo
*/
public static native int lsb_modreservation(modRsvRequest request);
@ -19291,11 +19291,11 @@ public static class ByValue extends jobArrayElementLog implements Structure.ByVa
* <b>Define Statements:</b>
* \par
* none
* @see \ref lsb_closestream
* @see \ref lsb_readstreamline
* @see \ref lsb_writestream
* @see \ref lsb_readstream
* @see \ref lsb_streamversion
* #see \ref lsb_closestream
* #see \ref lsb_readstreamline
* #see \ref lsb_writestream
* #see \ref lsb_readstream
* #see \ref lsb_streamversion
*/
// NOTE: Not in libbat
//public static native int lsb_openstream(lsbStream params);
@ -19335,11 +19335,11 @@ public static class ByValue extends jobArrayElementLog implements Structure.ByVa
* <b>Define Statements:</b>
* \par
* none
* @see \ref lsb_openstream
* @see \ref lsb_readstreamline
* @see \ref lsb_writestream
* @see \ref lsb_readstream
* @see \ref lsb_streamversion
* #see \ref lsb_openstream
* #see \ref lsb_readstreamline
* #see \ref lsb_writestream
* #see \ref lsb_readstream
* #see \ref lsb_streamversion
*/
// NOTE: Not in libbat
//public static native int lsb_closestream(String config);
@ -19379,13 +19379,13 @@ public static class ByValue extends jobArrayElementLog implements Structure.ByVa
* <b>Files:</b>
* \par
* $LSB_CONFDIR/cluster_name/configdir/lsb.params
* @see \ref lsb_closestream
* @see \ref lsb_geteventrec
* @see \ref lsb_openstream
* @see \ref lsb_puteventrec
* @see \ref lsb_readstreamline
* @see \ref lsb_writestream
* @see \ref lsb_readstream
* #see \ref lsb_closestream
* #see \ref lsb_geteventrec
* #see \ref lsb_openstream
* #see \ref lsb_puteventrec
* #see \ref lsb_readstreamline
* #see \ref lsb_writestream
* #see \ref lsb_readstream
*/
// NOTE: Not in libbat
//public static native String lsb_streamversion();
@ -19488,13 +19488,13 @@ public static class ByValue extends jobArrayElementLog implements Structure.ByVa
* \n \ref defs_lsb_XF_OP
* \n \ref jobgroup_controltypes
* \n \ref signal_action
* @see \ref lsb_closestream
* @see \ref lsb_geteventrec
* @see \ref lsb_openstream
* @see \ref lsb_puteventrec
* @see \ref lsb_readstreamline
* @see \ref lsb_streamversion
* @see \ref lsb_readstream
* #see \ref lsb_closestream
* #see \ref lsb_geteventrec
* #see \ref lsb_openstream
* #see \ref lsb_puteventrec
* #see \ref lsb_readstreamline
* #see \ref lsb_streamversion
* #see \ref lsb_readstream
*/
// NOTE: Not in libbat
//public static native int lsb_writestream(eventRec logPtr);
@ -19532,13 +19532,13 @@ public static class ByValue extends jobArrayElementLog implements Structure.ByVa
* <b>Define Statements:</b>
* \par
* none
* @see \ref lsb_closestream
* @see \ref lsb_geteventrec
* @see \ref lsb_openstream
* @see \ref lsb_puteventrec
* @see \ref lsb_readstreamline
* @see \ref lsb_streamversion
* @see \ref lsb_writestream
* #see \ref lsb_closestream
* #see \ref lsb_geteventrec
* #see \ref lsb_openstream
* #see \ref lsb_puteventrec
* #see \ref lsb_readstreamline
* #see \ref lsb_streamversion
* #see \ref lsb_writestream
*/
// NOTE: Not in libbat
//public static native eventRec.ByReference lsb_readstream(IntByReference nline);
@ -19578,13 +19578,13 @@ public static class ByValue extends jobArrayElementLog implements Structure.ByVa
* <b>Define Statements:</b>
* \par
* none
* @see \ref lsb_closestream
* @see \ref lsb_geteventrec
* @see \ref lsb_openstream
* @see \ref lsb_puteventrec
* @see \ref lsb_readstream
* @see \ref lsb_streamversion
* @see \ref lsb_writestream
* #see \ref lsb_closestream
* #see \ref lsb_geteventrec
* #see \ref lsb_openstream
* #see \ref lsb_puteventrec
* #see \ref lsb_readstream
* #see \ref lsb_streamversion
* #see \ref lsb_writestream
*/
// NOTE: Not in libbat
//public static native eventRec.ByReference lsb_readstreamline(String line);
@ -19912,7 +19912,7 @@ public static class ByValue extends jobArrayElementLog implements Structure.ByVa
* <b>Files:</b>
* \par
* none
* @see \ref lsb_resize_release
* #see \ref lsb_resize_release
*/
public static native int lsb_resize_cancel(long jobId);
@ -19958,7 +19958,7 @@ public static class ByValue extends jobArrayElementLog implements Structure.ByVa
* <b>Define Statements:</b>
* \par
* \ref resizablejob_related
* @see \ref lsb_resize_cancel
* #see \ref lsb_resize_cancel
*/
public static native int lsb_resize_release(job_resize_release req);

9
scala/qscript/examples/HelloWorld.scala 100644
View File

@ -0,0 +1,9 @@
import org.broadinstitute.sting.queue.QScript
class HelloWorld extends QScript {
def script = {
add(new CommandLineFunction {
def commandLine = "echo hello world"
})
}
}

24
scala/src/org/broadinstitute/sting/queue/engine/DispatchJobRunner.scala → scala/src/org/broadinstitute/sting/queue/engine/CommandLineJobRunner.scala
View File

@ -5,9 +5,29 @@ import org.broadinstitute.sting.queue.util.IOUtils
import java.io.File
/**
* Dispatches jobs to a compute cluster.
* Runs a command line function.
*/
trait DispatchJobRunner extends JobRunner[CommandLineFunction] {
trait CommandLineJobRunner extends JobRunner[CommandLineFunction] {
/** A generated exec shell script. */
protected var exec: File = _
/** Which directory to use for the job status files. */
protected def jobStatusDir = function.jobTempDir
/**
* Writes the function command line to an exec file.
*/
protected def writeExec() {
this.exec = IOUtils.writeTempFile(function.commandLine, ".exec", "", jobStatusDir)
}
/**
* Removes all temporary files used for this LSF job.
*/
def removeTemporaryFiles() = {
IOUtils.tryDelete(exec)
}
/**
* Builds a command line that can be run to force an automount of the directories.
* @param function Function to look jobDirectories.

20
scala/src/org/broadinstitute/sting/queue/engine/JobRunner.scala
View File

@ -28,19 +28,25 @@ trait JobRunner[TFunction <: QFunction] {
*/
def function: TFunction
protected def writeDone() = {
/**
* Writes the basic function description to the job done file.
*/
protected def writeDone() {
val content = "%s%nDone.".format(function.description)
IOUtils.writeContents(function.jobOutputFile, content)
}
protected def writeError(content: String) = {
/**
* Writes the contents of the error to the error file.
*/
protected def writeError(content: String) {
IOUtils.writeContents(functionErrorFile, content)
}
/**
* Writes the stack trace to the error file.
*/
protected def writeStackTrace(e: Throwable) = {
protected def writeStackTrace(e: Throwable) {
val stackTrace = new StringWriter
val printWriter = new PrintWriter(stackTrace)
printWriter.println(function.description)
@ -53,12 +59,16 @@ trait JobRunner[TFunction <: QFunction] {
* Calls back to a hook that an expert user can setup to modify a job.
* @param value Value to modify.
*/
protected def updateJobRun(value: Any) = {
protected def updateJobRun(value: Any) {
val updater = function.updateJobRun
if (updater != null)
if (updater.isDefinedAt(value))
updater(value)
}
private def functionErrorFile = if (function.jobErrorFile != null) function.jobErrorFile else function.jobOutputFile
/**
* Returns the path to the file to use for logging errors.
* @return the path to the file to use for logging errors.
*/
protected def functionErrorFile = if (function.jobErrorFile != null) function.jobErrorFile else function.jobOutputFile
}

6
scala/src/org/broadinstitute/sting/queue/engine/Lsf706JobRunner.scala
View File

@ -63,14 +63,14 @@ class Lsf706JobRunner(function: CommandLineFunction) extends LsfJobRunner(functi
request.options |= LibBat.SUB_JOB_NAME
}
exec = writeExec()
writeExec()
request.command = "sh " + exec
preExec = writePreExec()
writePreExec()
request.preExecCmd = "sh " + preExec
request.options |= LibBat.SUB_PRE_EXEC
postExec = writePostExec()
writePostExec()
request.postExecCmd = "sh " + postExec
request.options3 |= LibBat.SUB3_POST_EXEC

33
scala/src/org/broadinstitute/sting/queue/engine/LsfJobRunner.scala
View File

@ -7,14 +7,11 @@ import org.broadinstitute.sting.queue.util._
/**
* Runs jobs on an LSF compute cluster.
*/
abstract class LsfJobRunner(val function: CommandLineFunction) extends DispatchJobRunner with Logging {
abstract class LsfJobRunner(val function: CommandLineFunction) extends CommandLineJobRunner with Logging {
protected var runStatus: RunnerStatus.Value = _
var jobId = -1L
/** Which directory to use for the job status files. */
protected def jobStatusDir = function.jobTempDir
/** A file to look for to validate that the function ran to completion. */
protected var jobStatusPath: String = _
@ -24,9 +21,6 @@ abstract class LsfJobRunner(val function: CommandLineFunction) extends DispatchJ
/** A temporary job done file to let Queue know that the process exited with an error. */
private lazy val jobFailFile = new File(jobStatusPath + ".fail")
/** A generated exec shell script. */
protected var exec: File = _
/** A generated pre-exec shell script. */
protected var preExec: File = _
@ -80,8 +74,8 @@ abstract class LsfJobRunner(val function: CommandLineFunction) extends DispatchJ
/**
* Removes all temporary files used for this LSF job.
*/
def removeTemporaryFiles() = {
IOUtils.tryDelete(exec)
override def removeTemporaryFiles() = {
super.removeTemporaryFiles()
IOUtils.tryDelete(preExec)
IOUtils.tryDelete(postExec)
IOUtils.tryDelete(jobDoneFile)
@ -92,7 +86,7 @@ abstract class LsfJobRunner(val function: CommandLineFunction) extends DispatchJ
* Outputs the last lines of the error logs.
*/
protected def tailError() = {
val errorFile = if (function.jobErrorFile != null) function.jobErrorFile else function.jobOutputFile
val errorFile = functionErrorFile
if (IOUtils.waitFor(errorFile, 120)) {
val tailLines = IOUtils.tail(errorFile, 100)
val nl = "%n".format()
@ -102,21 +96,11 @@ abstract class LsfJobRunner(val function: CommandLineFunction) extends DispatchJ
}
}
/**
* Writes an exec file to cleanup any status files and
* optionally mount any automount directories on the node.
* @return the file path to the pre-exec.
*/
protected def writeExec() = {
IOUtils.writeTempFile(function.commandLine, ".exec", "", jobStatusDir)
}
/**
* Writes a pre-exec file to cleanup any status files and
* optionally mount any automount directories on the node.
* @return the file path to the pre-exec.
*/
protected def writePreExec() = {
protected def writePreExec() {
val preExec = new StringBuilder
preExec.append("rm -f '%s/'.$LSB_JOBID.done%n".format(jobStatusDir))
@ -127,14 +111,13 @@ abstract class LsfJobRunner(val function: CommandLineFunction) extends DispatchJ
mountCommand(function).foreach(command =>
preExec.append("%s%n".format(command)))
IOUtils.writeTempFile(preExec.toString, ".preExec", "", jobStatusDir)
this.preExec = IOUtils.writeTempFile(preExec.toString, ".preExec", "", jobStatusDir)
}
/**
* Writes a post-exec file to create the status files.
* @return the file path to the post-exec.
*/
protected def writePostExec() = {
protected def writePostExec() {
val postExec = new StringBuilder
val touchDone = function.doneOutputs.map("touch '%s'%n".format(_)).mkString
@ -153,6 +136,6 @@ abstract class LsfJobRunner(val function: CommandLineFunction) extends DispatchJ
|fi
|""".stripMargin.format(jobStatusDir, touchDone, touchFail))
IOUtils.writeTempFile(postExec.toString, ".postExec", "", jobStatusDir)
this.postExec = IOUtils.writeTempFile(postExec.toString, ".postExec", "", jobStatusDir)
}
}

10
scala/src/org/broadinstitute/sting/queue/engine/ShellJobRunner.scala
View File

@ -6,7 +6,7 @@ import org.broadinstitute.sting.queue.util.{JobExitException, Logging, ShellJob}
/**
* Runs jobs one at a time locally
*/
class ShellJobRunner(val function: CommandLineFunction) extends JobRunner[CommandLineFunction] with Logging {
class ShellJobRunner(val function: CommandLineFunction) extends CommandLineJobRunner with Logging {
private var runStatus: RunnerStatus.Value = _
/**
@ -16,11 +16,14 @@ class ShellJobRunner(val function: CommandLineFunction) extends JobRunner[Comman
def start() = {
try {
val job = new ShellJob
job.command = function.commandLine
job.workingDir = function.commandDirectory
job.outputFile = function.jobOutputFile
job.errorFile = function.jobErrorFile
writeExec()
job.shellScript = exec
// Allow advanced users to update the job.
updateJobRun(job)
@ -39,6 +42,7 @@ class ShellJobRunner(val function: CommandLineFunction) extends JobRunner[Comman
function.mkOutputDirectories()
runStatus = RunnerStatus.RUNNING
job.run()
removeTemporaryFiles()
function.doneOutputs.foreach(_.createNewFile())
runStatus = RunnerStatus.DONE
logger.info("Done: " + function.commandLine)
@ -46,6 +50,7 @@ class ShellJobRunner(val function: CommandLineFunction) extends JobRunner[Comman
case jee: JobExitException =>
runStatus = RunnerStatus.FAILED
try {
removeTemporaryFiles()
function.failOutputs.foreach(_.createNewFile())
writeError(jee.getMessage)
} catch {
@ -56,6 +61,7 @@ class ShellJobRunner(val function: CommandLineFunction) extends JobRunner[Comman
case e =>
runStatus = RunnerStatus.FAILED
try {
removeTemporaryFiles()
function.failOutputs.foreach(_.createNewFile())
writeStackTrace(e)
} catch {

2
scala/src/org/broadinstitute/sting/queue/util/CommandLineJob.scala
View File

@ -6,7 +6,7 @@ import java.io.File
* Base utility class for a command line job.
*/
abstract class CommandLineJob {
var command: String = _
var shellScript: File = _
var workingDir: File = _
var inputFile: File = _
var outputFile: File = _

10
scala/src/org/broadinstitute/sting/queue/util/IOUtils.scala
View File

@ -48,11 +48,19 @@ object IOUtils extends Logging {
*/
def writeContents(file: File, content: String) = FileUtils.writeStringToFile(file, content)
/**
* Reads content of a file into a string.
* Only for use on really small files!
* @param file File to read to.
* @return content Content of the file.
*/
def readContents(file: File) = FileUtils.readFileToString(file)
/**
* Writes content to a temp file and returns the path to the temporary file.
* @param content to write.
* @param prefix Prefix for the temp file.
* @parm suffix Suffix for the temp file.
* @param suffix Suffix for the temp file.
* @param directory Directory for the temp file.
* @return the path to the temp file.
*/

6
scala/src/org/broadinstitute/sting/queue/util/ShellJob.scala
View File

@ -1,7 +1,5 @@
package org.broadinstitute.sting.queue.util
import org.broadinstitute.sting.queue.QException
/**
* Runs a job on the command line by invoking "sh -c <command>"
*/
@ -10,14 +8,12 @@ class ShellJob extends CommandLineJob with Logging {
* Runs the command and waits for the output.
*/
def run() = {
assert(command != null, "Command was not set on job")
val (redirectError, errorFile) = if (this.errorFile == null) (true, null) else (false, this.errorFile)
val bufferSize = if (redirectError || logger.isDebugEnabled) FIVE_MB else 0
val stdinSettings = new ProcessController.InputStreamSettings(null, this.inputFile)
val stdoutSettings = new ProcessController.OutputStreamSettings(bufferSize, this.outputFile, true)
val stderrSettings = new ProcessController.OutputStreamSettings(FIVE_MB, errorFile, true)
val commandLine = Array("sh", "-c", command)
val commandLine = Array("sh", shellScript.toString)
val processSettings = new ProcessController.ProcessSettings(
commandLine, null, this.workingDir, stdinSettings, stdoutSettings, stderrSettings, redirectError)

76
scala/test/org/broadinstitute/sting/queue/util/ShellJobUnitTest.scala 100644
View File

@ -0,0 +1,76 @@
package org.broadinstitute.sting.queue.util
import org.broadinstitute.sting.BaseTest
import org.testng.annotations.Test
import java.io.File
import org.testng.Assert
class ShellJobUnitTest extends BaseTest {
@Test
def testEcho {
val job = new ShellJob
job.shellScript = writeScript("echo Hello World")
job.run()
}
@Test(expectedExceptions=Array(classOf[JobExitException]))
def testBadQuotes {
val job = new ShellJob
job.shellScript = writeScript("echo 'Hello World")
job.run()
}
@Test
def testGoodQuotes {
val job = new ShellJob
job.shellScript = writeScript("echo 'Hello World'")
job.run()
}
@Test
def testEscapeCharacters {
var job: ShellJob = null
job = new ShellJob
job.shellScript = writeScript("echo #")
job.outputFile = File.createTempFile("temp", "")
job.run()
Assert.assertEquals(IOUtils.readContents(job.outputFile).trim, "")
job = new ShellJob
job.shellScript = writeScript("""echo \#""")
job.outputFile = File.createTempFile("temp", "")
job.run()
Assert.assertEquals(IOUtils.readContents(job.outputFile).trim, "#")
job = new ShellJob
job.shellScript = writeScript("""echo \\#""")
job.outputFile = File.createTempFile("temp", "")
job.run()
Assert.assertEquals(IOUtils.readContents(job.outputFile).trim, """\#""")
}
@Test
def testLongCommand {
// This command fails on some systems with a 4096 character limit when run via the old sh -c "echo ...",
// but works on the same systems when run via sh <script>
val builder = new StringBuilder
builder.append("echo ")
for (i <- 1 to 500) {
val s = i.toString
builder.append("000".take(3-s.length)).append(s).append("______ ")
}
val job = new ShellJob
job.shellScript = writeScript(builder.toString)
job.run()
}
private val tempDir = new File(System.getProperty("java.io.tmpdir"))
private def writeScript(contents: String) = {
val script = IOUtils.writeTempFile(contents, "temp", "", tempDir)
script.deleteOnExit
script
}
}