From 429800a1929be9740b6864eef87ef4d1f525de69 Mon Sep 17 00:00:00 2001 From: Guillermo del Angel Date: Wed, 2 May 2012 09:57:06 -0400 Subject: [PATCH] Fix corner case rounding issue in MathUtils unit test: 10^logFactorial(4)) was 23.999999... which if cast directly yielded 23 - so, do pre-rounding to ensure correct integer result if caller will cast value. --- public/java/src/org/broadinstitute/sting/utils/MathUtils.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/public/java/src/org/broadinstitute/sting/utils/MathUtils.java b/public/java/src/org/broadinstitute/sting/utils/MathUtils.java index 242e40e6c..43861261c 100644 --- a/public/java/src/org/broadinstitute/sting/utils/MathUtils.java +++ b/public/java/src/org/broadinstitute/sting/utils/MathUtils.java @@ -1463,7 +1463,8 @@ public class MathUtils { } public static double factorial(int x) { - return Math.pow(10, log10Factorial(x)); + // avoid rounding errors caused by fact that 10^log(x) might be slightly lower than x and flooring may produce 1 less than real value + return (double)Math.round(Math.pow(10, log10Factorial(x))); } public static double log10Factorial(int x) {