From 985daec76e31b1f2bad0e12ec3a4ead296e1cf47 Mon Sep 17 00:00:00 2001
From: rpoplin <rpoplin@348d0f76-0448-11de-a6fe-93d51630548a>
Date: Fri, 4 Dec 2009 14:24:49 +0000
Subject: [PATCH] Fixed problem with integer overflow in R scripts.

git-svn-id: file:///humgen/gsa-scr1/gsa-engineering/svn_contents/trunk@2258 348d0f76-0448-11de-a6fe-93d51630548a
---
 R/plot_residualError_OtherCovariate.R        | 4 ++--
 R/plot_residualError_QualityScoreCovariate.R | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/R/plot_residualError_OtherCovariate.R b/R/plot_residualError_OtherCovariate.R
index ba187036e..23bdd7e1f 100644
--- a/R/plot_residualError_OtherCovariate.R
+++ b/R/plot_residualError_OtherCovariate.R
@@ -18,8 +18,8 @@ c <- c[sort.list(c[,1]),]
 
 d.good <- c[c$nBases >= 1000,]
 d.1000 <- c[c$nBases < 1000,]
-rmseGood = sqrt(sum((d.good$Qempirical-d.good$Qreported)^2 * d.good$nBases) / sum(d.good$nBases) )
-rmseAll = sqrt(sum((c$Qempirical-c$Qreported)^2 * c$nBases) / sum(c$nBases) )
+rmseGood = sqrt( sum(as.numeric((d.good$Qempirical-d.good$Qreported)^2 * d.good$nBases)) / sum(as.numeric(d.good$nBases)) ) # prevent integer overflow with as.numeric, ugh
+rmseAll = sqrt( sum(as.numeric((c$Qempirical-c$Qreported)^2 * c$nBases)) / sum(as.numeric(c$nBases)) )
 theTitle = paste("RMSE_good = ", round(rmseGood,digits=3), ", RMSE_all = ", round(rmseAll,digits=3))
 if( length(d.good$nBases) == length(c$nBases) ) {
 	theTitle = paste("RMSE = ", round(rmseAll,digits=3))
diff --git a/R/plot_residualError_QualityScoreCovariate.R b/R/plot_residualError_QualityScoreCovariate.R
index 7dc885205..00436c8b3 100644
--- a/R/plot_residualError_QualityScoreCovariate.R
+++ b/R/plot_residualError_QualityScoreCovariate.R
@@ -18,8 +18,8 @@ d.1000 <- t[t$nBases < 1000  & t$Qreported >= Qcutoff,]
 d.10000 <- t[t$nBases < 10000 & t$nBases >= 1000  & t$Qreported >= Qcutoff,]
 f <- t[t$Qreported < Qcutoff,]
 e <- rbind(d.good, d.1000, d.10000)
-rmseGood = sqrt(sum((d.good$Qempirical-d.good$Qreported)^2 * d.good$nBases) / sum(d.good$nBases) )
-rmseAll = sqrt(sum((e$Qempirical-e$Qreported)^2 * e$nBases) / sum(e$nBases) )
+rmseGood = sqrt( sum(as.numeric((d.good$Qempirical-d.good$Qreported)^2 * d.good$nBases)) / sum(as.numeric(d.good$nBases)) ) # prevent integer overflow with as.numeric, ugh
+rmseAll = sqrt( sum(as.numeric((e$Qempirical-e$Qreported)^2 * e$nBases)) / sum(as.numeric(e$nBases)) )
 theTitle = paste("RMSE_good = ", round(rmseGood,digits=3), ", RMSE_all = ", round(rmseAll,digits=3))
 if( length(t$nBases) - length(f$nBases) == length(d.good$nBases) ) {
 	theTitle = paste("RMSE = ", round(rmseAll,digits=3));