adding some utility code I've found helpful when working the Tribble index code
git-svn-id: file:///humgen/gsa-scr1/gsa-engineering/svn_contents/trunk@4108 348d0f76-0448-11de-a6fe-93d51630548a
This commit is contained in:
aaron
parent
4eff69d95e
commit
bc0826801c
|
|
@ -0,0 +1,9 @@
|
|||
This file is a roadmap to the contents of the Ruby directory
|
||||
|
||||
Contents:
|
||||
------------------------------------------------------------------------------------
|
||||
restartBamboo.rb - a script used to restart Bamboo after system failures
|
||||
validateIndex.rb - a script for working with Tribble indexes, type 'validateIndex.rb' -h for help
|
||||
|
||||
utils/ - any basic utility methods
|
||||
index/ - the index utility code
|
||||
|
|
@ -0,0 +1,70 @@
|
|||
require './util/BinaryFileReader'
|
||||
# this base class for all index types (at least linear and tree)
|
||||
class Index
|
||||
attr_reader :type, :headerVersion, :fileName, :fileSize, :t5, :md5, :flags
|
||||
|
||||
# construct, given a file name
|
||||
def initialize(fileName)
|
||||
@inputFile = fileName
|
||||
@file = BinaryFileReader.new(fileName)
|
||||
magic = @file.readBytes(4)
|
||||
if (magic != "TIDX")
|
||||
print "#{@inputFile}: !! Magic number is not what we expected, TIDX, instead we saw #{magic} !!\n"
|
||||
exit(1)
|
||||
end
|
||||
@type = @file.readInt()
|
||||
@headerVersion = @file.readInt()
|
||||
@fileName = @file.readString()
|
||||
@fileSize = @file.readLong()
|
||||
@ts = @file.readLong()
|
||||
@md5 = @file.readString()
|
||||
@flags = @file.readUInt()
|
||||
@seqDict = readSeqDictionary() if (@flags == 32768)
|
||||
@propCount = readPropertyDictionary() if (@headerVersion >= 3)
|
||||
end
|
||||
|
||||
def validate()
|
||||
f = Proc.new{ print "#{@inputFile}:\t\terror: invalid type, we saw #{@type} but expected [1-2]\n"; return false} if @type < 1 or @type > 2
|
||||
f = Proc.new{ print "#{@inputFile}:\t\terror: invalid header version, we saw #{@headerVersion} but expected [1-3]\n"; return false} if @headerVersion < 1 or @headerVersion > 3
|
||||
f = Proc.new{ print "#{@inputFile}:\t\twarning: on fileName, we saw '#{@fileName}' but expected actual text\n"; return false} if @fileName == ""
|
||||
f = Proc.new{ print "#{@inputFile}:\t\twarning: on TS, we saw '#{@ts}' but expected actual text\n"; return false} if @ts == ""
|
||||
f = Proc.new{ print "#{@inputFile}:\t\twarning: on md5, we saw '#{@md5}' but expected actual text\n"; return false} if @md5 == ""
|
||||
f.call if f != nil
|
||||
return true
|
||||
end
|
||||
|
||||
# diff two headers
|
||||
def diffHeader(otherIndex)
|
||||
self.instance_variables.each { |var|
|
||||
next if "#{var}" == "@file" or "#{var}" == "@sequences"
|
||||
puts "Other header doesn't define #{var}" if !(otherIndex.instance_variable_defined?(var))
|
||||
one = (self.instance_variable_get(var)).to_s
|
||||
two = (otherIndex.instance_variable_get(var)).to_s
|
||||
puts "type #{var} not equal, #{one} != #{two}" if one != two
|
||||
}
|
||||
end
|
||||
|
||||
# read the sequence dictionary, assuming we have one
|
||||
def readSeqDictionary()
|
||||
sequences = []
|
||||
count = @file.readInt()
|
||||
count.times {|index|
|
||||
sequences.add(@file.readString())
|
||||
@file.readInt() # drop the sizes for now
|
||||
}
|
||||
sequences # return sequences
|
||||
end
|
||||
|
||||
# read the sequence dictionary, assuming we have one
|
||||
def readPropertyDictionary()
|
||||
sequences = {}
|
||||
count = @file.readInt()
|
||||
count.times {|index|
|
||||
sequences.put(@file.readString(),@file.readString()) }
|
||||
sequences # return sequences
|
||||
end
|
||||
|
||||
def close()
|
||||
@file.close()
|
||||
end
|
||||
end
|
||||
|
|
@ -0,0 +1,41 @@
|
|||
# the implementation of the interval index class
|
||||
$LOAD_PATH << File.dirname(__FILE__)
|
||||
require "Index.rb"
|
||||
|
||||
class IntervalIndex < Index
|
||||
def initialize(file)
|
||||
super(file)
|
||||
@nSeq = @file.readInt()
|
||||
@sequences = Array.new()
|
||||
@nSeq.times {|index|
|
||||
@sequences.push(TISeqEntry.new(@file))
|
||||
}
|
||||
end
|
||||
|
||||
def diff(otherIndex)
|
||||
diffHeader(otherIndex)
|
||||
if (otherIndex.type != @type)
|
||||
print "Indexes are not the same type (#{otherIndex.type} != #{@type})\n"
|
||||
return false
|
||||
end
|
||||
ret = false
|
||||
end
|
||||
end
|
||||
|
||||
class TISeqEntry
|
||||
def initialize(file)
|
||||
@contig = file.readString()
|
||||
@binCount = file.readInt()
|
||||
@startPositions = Array.new()
|
||||
@endPositions = Array.new()
|
||||
@positions = Array.new()
|
||||
@sizes = Array.new()
|
||||
@binCount.times { |index|
|
||||
@startPositions.push(file.readInt())
|
||||
@endPositions.push(file.readInt())
|
||||
@positions.push(file.readLong())
|
||||
@sizes.push(file.readInt())
|
||||
}
|
||||
end
|
||||
end
|
||||
|
||||
|
|
@ -0,0 +1,59 @@
|
|||
# the linear index class implementation
|
||||
$LOAD_PATH << File.dirname(__FILE__)
|
||||
require "Index.rb"
|
||||
|
||||
class LinearIndex < Index
|
||||
attr_accessor :nSeq, :sequences
|
||||
def initialize(file)
|
||||
super(file)
|
||||
@nSeq = @file.readInt()
|
||||
@sequences = Array.new()
|
||||
@nSeq.times {|index|
|
||||
@sequences.push(LISeqEntry.new(@file))
|
||||
}
|
||||
end
|
||||
|
||||
def diff(otherIndex)
|
||||
diffHeader(otherIndex)
|
||||
if (otherIndex.type != @type)
|
||||
print "Indexes are not the same type (#{otherIndex.type} != #{@type})\n"
|
||||
return false
|
||||
end
|
||||
ret = false
|
||||
notInOther = @sequences.reject {|item|
|
||||
return true if !otherIndex.sequences.include?(item)
|
||||
item.diff(otherIndex.sequences[otherIndex.sequences.index(item)])
|
||||
}
|
||||
notInOther.pretty_print
|
||||
end
|
||||
|
||||
|
||||
end
|
||||
|
||||
class LISeqEntry
|
||||
def initialize(file)
|
||||
@contig = file.readString()
|
||||
@binWidth = file.readInt()
|
||||
@binCount = file.readInt()
|
||||
@longestFeature = file.readInt()
|
||||
@maxBin = file.readInt()
|
||||
@totalBin = file.readInt()
|
||||
@startPositions = Array.new()
|
||||
@binCount.times { |index|
|
||||
@startPositions.push(file.readLong())
|
||||
}
|
||||
@finalPos = file.readLong()
|
||||
end
|
||||
|
||||
# print a summary of the index characteristics
|
||||
def diff(otherLISeqEntry)
|
||||
self.instance_variables.each { |var|
|
||||
next if "#{var}" == "@file" or "#{var}" == "@sequences"
|
||||
puts "Other LISeqEntry doesn't define #{var}" if !(otherLISeqEntry.instance_variable_defined?(var))
|
||||
one = (self.instance_variable_get(var)).to_s
|
||||
two = (otherLISeqEntry.instance_variable_get(var)).to_s
|
||||
puts "otherLISeqEntry: type #{var} not equal, #{one} != #{two}" if one != two
|
||||
}
|
||||
end
|
||||
end
|
||||
|
||||
|
|
@ -0,0 +1,170 @@
|
|||
# this ruby files takes two indexes (of the same type) and diff's them. If they're different types,
|
||||
# it'll stop after the header
|
||||
|
||||
# a function to exit, printing a message
|
||||
def exitWithError(message)
|
||||
puts ###########################################
|
||||
puts message
|
||||
puts ###########################################
|
||||
exit(1)
|
||||
end
|
||||
|
||||
|
||||
if (ARGV.size != 2)
|
||||
exitWithError("We take two files as input, try again!")
|
||||
end
|
||||
|
||||
# open the indexes
|
||||
index1 = File.new(ARGV[0])
|
||||
index2 = File.new(ARGV[1])
|
||||
|
||||
# a helper function for comparing values
|
||||
def compValues(file1, file2, byteCount, type)
|
||||
index1Type = file1.sysread(byteCount)
|
||||
index2Type = file2.sysread(byteCount)
|
||||
if (index1Type != index2Type)
|
||||
print "#{type}, index1 (#{index1Type}) != index2 (#{index2Type})\n"
|
||||
else
|
||||
print "#{type}, index1 (#{index1Type}) == index2 (#{index2Type})\n"
|
||||
end
|
||||
end
|
||||
|
||||
# a helper function for comparing values
|
||||
def compInts(file1, file2, byteCount, type)
|
||||
if (byteCount > 4)
|
||||
upack = "q"
|
||||
else
|
||||
upack = "i"
|
||||
end
|
||||
index1Type = file1.sysread(byteCount).unpack(upack)
|
||||
index2Type = file2.sysread(byteCount).unpack(upack)
|
||||
if (index1Type != index2Type)
|
||||
print "#{type}, index1 (#{index1Type}) != index2 (#{index2Type})\n"
|
||||
else
|
||||
print "#{type}, index1 (#{index1Type}) == index2 (#{index2Type})\n"
|
||||
end
|
||||
end
|
||||
|
||||
|
||||
# a helper function for reading strings
|
||||
def readString(index)
|
||||
buffer = []
|
||||
ch = index.sysread(1)
|
||||
while (ch != "\0")
|
||||
buffer.push(ch)
|
||||
ch = index.sysread(1)
|
||||
end
|
||||
buffer.to_s
|
||||
end
|
||||
|
||||
# validate the magic number from both
|
||||
exitWithError("Magic number not valid for index 1") if index1.sysread(4) != "TIDX"
|
||||
exitWithError("Magic number not valid for index 2") if index2.sysread(4) != "TIDX"
|
||||
|
||||
# validate the types
|
||||
compInts(index1,index2,4,"types")
|
||||
|
||||
# validate the versions
|
||||
v1 = index1.sysread(4).unpack("i")
|
||||
v2 = index2.sysread(4).unpack("i")
|
||||
if (v1 != v2)
|
||||
print "version, index1 (#{v1}) != index2 (#{v2})\n"
|
||||
else
|
||||
print "version, index1 (#{v1}) == index2 (#{v2})\n"
|
||||
end
|
||||
|
||||
# validate the filenames
|
||||
fl1 = readString(index1)
|
||||
fl2 = readString(index2)
|
||||
if (fl1 != fl2)
|
||||
print "filename, index1 (#{fl1}) != index2 (#{fl2})\n"
|
||||
else
|
||||
print "filename, index1 (#{fl1}) == index2 (#{fl2})\n"
|
||||
end
|
||||
# validate the sizes
|
||||
compInts(index1,index2,8,"sizes")
|
||||
|
||||
|
||||
# validate the T5?
|
||||
compValues(index1,index2,8,"T5")
|
||||
|
||||
# validate the MD5 - just a byte, we don't write the MD5 sums in yet
|
||||
# validate the filenames
|
||||
fl1 = readString(index1)
|
||||
fl2 = readString(index2)
|
||||
if (fl1 != fl2)
|
||||
print "md5, index1 (#{fl1}) != index2 (#{fl2})\n"
|
||||
else
|
||||
print "md5, index1 (#{fl1}) == index2 (#{fl2})\n"
|
||||
end
|
||||
|
||||
# validate the flags
|
||||
index1Flags = (index1.sysread(4)).unpack("L")
|
||||
index2Flags = (index2.sysread(4)).unpack("L")
|
||||
if (index1Flags != index2Flags)
|
||||
print "Flags are different, index1 = #{index1Flags[0]}, index2 = #{index2Flags[0]}\n"
|
||||
end
|
||||
|
||||
def readSeqDictionary(file)
|
||||
puts "reading seq dict"
|
||||
sequences = []
|
||||
count = (file.sysread(4)).unpack("i")
|
||||
puts count
|
||||
count[0].times {|index|
|
||||
sequences.add(readString(file).to_s)
|
||||
file.sysread(4) # drop the sizes for now
|
||||
}
|
||||
sequences # return sequences
|
||||
end
|
||||
|
||||
if (index1Flags[0] == 32768)
|
||||
puts readSeqDictionary(index1)
|
||||
elsif (index2Flags[0] == 32768)
|
||||
puts readSeqDictionary(index2)
|
||||
end
|
||||
|
||||
# bump off the prop dictionary
|
||||
index1.sysread(4) if (v1[0] == 3)
|
||||
index2.sysread(4) if (v2[0] == 3)
|
||||
|
||||
def readSeqEntry(i1)
|
||||
puts "--------------------------------------------"
|
||||
puts "Contig --> #{readString(i1).to_s}"
|
||||
print "bin width = #{(i1.sysread(4)).unpack("i")}\n"
|
||||
binCount = (i1.sysread(4)).unpack("i")[0]
|
||||
print "number of bins = #{binCount}\n"
|
||||
print "longest feature = #{(i1.sysread(4)).unpack("i")}\n"
|
||||
print "max bin size = #{(i1.sysread(4)).unpack("i")}\n"
|
||||
print "total bin size = #{(i1.sysread(4)).unpack("i")}\n"
|
||||
lastStartPos = -1
|
||||
binCount.times { |index|
|
||||
startPos = (i1.sysread(8)).unpack("q")[0]
|
||||
#if (startPos < lastStartPos)
|
||||
puts "bin at index #{index}, this start = #{startPos}, last start = #{lastStartPos}"
|
||||
#end
|
||||
lastStartPos = startPos
|
||||
}
|
||||
|
||||
end
|
||||
|
||||
def compContigLinear(i1, i2)
|
||||
seqC1 = (i1.sysread(4)).unpack("L")
|
||||
seqC2 = (i2.sysread(4)).unpack("L")
|
||||
print "seq count 1 = #{seqC1}, count 2 = #{seqC2}\n"
|
||||
puts "\nentries for index 1"
|
||||
seqC1[0].times { |index|
|
||||
readSeqEntry(i1)
|
||||
}
|
||||
puts "\nentries for index 2"
|
||||
seqC2[0].times { |index|
|
||||
readSeqEntry(i2)
|
||||
}
|
||||
end
|
||||
|
||||
compContigLinear(index1,index2)
|
||||
|
||||
|
||||
print "Done!\n"
|
||||
# close the files
|
||||
index1.close()
|
||||
index2.close()
|
||||
|
|
@ -0,0 +1,44 @@
|
|||
# a ruby class for reading in binary files; really this just adds some conv. methods like readInt(), readLong(), etc.
|
||||
class BinaryFileReader
|
||||
# constructor
|
||||
def initialize(fileName)
|
||||
@file = File.open(fileName,"r")
|
||||
end
|
||||
|
||||
# read and return an int (4 byte, signed, machine based endian)
|
||||
def readInt()
|
||||
(@file.sysread(4)).unpack("i")[0]
|
||||
end
|
||||
|
||||
# read and return an int (4 byte, unsigned, machine based endian)
|
||||
def readUInt()
|
||||
(@file.sysread(4)).unpack("L")[0]
|
||||
end
|
||||
|
||||
# read and return an long (8 byte, signed, machine based endian)
|
||||
def readLong()
|
||||
(@file.sysread(8)).unpack("q")[0]
|
||||
end
|
||||
|
||||
# read and return a set number of bytes as a string
|
||||
def readBytes(count)
|
||||
(@file.sysread(count)).to_s
|
||||
end
|
||||
|
||||
# read and return a null terminated string
|
||||
def readString()
|
||||
buffer = []
|
||||
ch = @file.sysread(1)
|
||||
while (ch != "\0")
|
||||
buffer.push(ch)
|
||||
ch = @file.sysread(1)
|
||||
end
|
||||
buffer.to_s
|
||||
end
|
||||
|
||||
# close the file
|
||||
def close()
|
||||
@file.close()
|
||||
end
|
||||
end
|
||||
|
||||
|
|
@ -0,0 +1,103 @@
|
|||
# this ruby files validates a linear index
|
||||
# set the include path to include the current directory
|
||||
$LOAD_PATH << File.dirname(__FILE__)
|
||||
|
||||
# require a couple of files
|
||||
require "index/Index.rb"
|
||||
require "index/LinearIndex.rb"
|
||||
require "index/IntervalIndex.rb"
|
||||
require "optparse"
|
||||
require "yaml"
|
||||
|
||||
# This hash will hold all of the options
|
||||
# parsed from the command-line by
|
||||
# OptionParser.
|
||||
options = {}
|
||||
|
||||
optparse = OptionParser.new do|opts|
|
||||
# Set a banner, displayed at the top
|
||||
# of the help screen.
|
||||
opts.banner = "Usage: ruby validateIndex.rb [options] file1 file2 ..."
|
||||
|
||||
# Define the options, and what they do
|
||||
options[:index] = [] if options[:index] == nil
|
||||
opts.on( '-i', '--index INDEX (REQUIRED)', 'Specify the index. Multiple are allowed' ) do |file| options[:index].push(file) end
|
||||
|
||||
options[:verbose] = false
|
||||
opts.on( '-v', '--verbose', 'Output more information' ) do options[:verbose] = true end
|
||||
|
||||
options[:validate] = false
|
||||
opts.on( '-c', '--check', 'Check (Validate) the index(es) passed in as parameters' ) do options[:check] = true end
|
||||
|
||||
options[:diff] = false
|
||||
opts.on( '-d', '--diff', 'Diff two indexes' ) do options[:diff] = true end
|
||||
|
||||
options[:print] = false
|
||||
opts.on( '-p', '--print', 'Print all of the information about the file' ) do options[:print] = true end
|
||||
|
||||
# This displays the help screen, all programs are
|
||||
# assumed to have this option.
|
||||
opts.on_tail( '-h', '--help', 'Display this screen' ) do
|
||||
puts opts
|
||||
exit
|
||||
end
|
||||
end
|
||||
# parse the command line
|
||||
optparse.parse!
|
||||
|
||||
#Now raise an exception if we have not found a host option
|
||||
if options[:index].size == 0
|
||||
puts "you must at least specify an index file!"
|
||||
puts optparse
|
||||
end
|
||||
|
||||
# a function to load an index
|
||||
def loadIndex(file)
|
||||
indexTry = Index.new(file)
|
||||
indexTry.close()
|
||||
if (indexTry.type == 1)
|
||||
puts "Linear index..."
|
||||
index = LinearIndex.new(file)
|
||||
else
|
||||
puts "Interval index..."
|
||||
index = IntervalIndex.new(file)
|
||||
end
|
||||
index
|
||||
end
|
||||
|
||||
#################### Control Block ####################
|
||||
|
||||
# load all of the indexes
|
||||
indexes = []
|
||||
options[:index].each {|indexFile|
|
||||
indexes.push(loadIndex(indexFile))
|
||||
}
|
||||
|
||||
# switch on the flags supplied
|
||||
if (options[:diff])
|
||||
if (options[:index].size != 2)
|
||||
print "Unable to diff indexes if you don't supply two and only two indexes\n";
|
||||
exit(1)
|
||||
else
|
||||
indexes[0].diff(indexes[1])
|
||||
end
|
||||
elsif (options[:validate])
|
||||
indexes.each {|index| index.validate() }
|
||||
elsif (options[:print])
|
||||
indexes.each {|index| puts YAML::dump( index ) }
|
||||
end
|
||||
|
||||
|
||||
|
||||
|
||||
# if they specified validate
|
||||
if (options[:check])
|
||||
options[:index].each {|index|
|
||||
idx = Index.new(index).validate()
|
||||
}
|
||||
end
|
||||
|
||||
|
||||
|
||||
|
||||
exit
|
||||
Loading…
Reference in New Issue