zzh
/
gatk-3.8
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Changing Uniform approximation to Normal approximation in rank sum test. n factorial was overflowing. 

git-svn-id: file:///humgen/gsa-scr1/gsa-engineering/svn_contents/trunk@5565 348d0f76-0448-11de-a6fe-93d51630548a
This commit is contained in:
rpoplin 2011-04-03 18:18:39 +00:00
parent b316c9a590
commit 79c43845ad
1 changed files with 2 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
java/src/org/broadinstitute/sting/utils/MannWhitneyU.java
View File

@ -92,7 +92,7 @@ public class MannWhitneyU {
} else if ( n > 8 || m > 8 ) { } else if ( n > 8 || m > 8 ) {
pval = calculatePFromTable(n,m,u); pval = calculatePFromTable(n,m,u);
} else { } else {
// small m and n [possibly small n] - full approx // small m and n - full approx
pval = calculatePRecursively(n,m,u); pval = calculatePRecursively(n,m,u);
} }
@ -102,7 +102,7 @@ public class MannWhitneyU {
public static double calculatePFromTable(int n, int m, long u) { public static double calculatePFromTable(int n, int m, long u) {
// todo -- actually use a table for: // todo -- actually use a table for:
// todo - n large, m small // todo - n large, m small
return calculatePUniformApproximation(n,m,u); return calculatePNormalApproximation(n,m,u);
} }
/** /**