issue #96: floating point precision Exception

kdbinsidebrains · Mar 13, 2024 · e258c81 · e258c81
1 parent 216b8a8
commit e258c81
Show file tree

Hide file tree

Showing 5 changed files with 29 additions and 9 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,5 +1,11 @@
 # KdbInsideBrains Changelog
 
+## [5.5.1]
+
+### Fixed
+
+- issue #96: floating point precision Exception
+
 ## [5.5.0]
 
 ### Added

diff --git a/src/main/java/org/kdb/inside/brains/view/KdbOutputFormatter.java b/src/main/java/org/kdb/inside/brains/view/KdbOutputFormatter.java
@@ -41,7 +41,7 @@ public final class KdbOutputFormatter {
 
         for (RoundingMode mode : RoundingMode.values()) {
             final int modeId = mode.ordinal();
-            for (int precision = 0; precision < NumericalOptions.MAX_DECIMAL_PRECISION; precision++) {
+            for (int precision = 0; precision <= NumericalOptions.MAX_DECIMAL_PRECISION; precision++) {
                 DECIMAL[modeId][precision] = new DecimalFormat("0." + "#".repeat(precision));
                 DECIMAL[modeId][precision].setRoundingMode(mode);
 

diff --git a/src/main/java/org/kdb/inside/brains/view/console/NumericalOptions.java b/src/main/java/org/kdb/inside/brains/view/console/NumericalOptions.java
@@ -16,6 +16,12 @@ public int getFloatPrecision() {
     }
 
     public void setFloatPrecision(int floatPrecision) {
+        if (floatPrecision < 0) {
+            throw new IllegalArgumentException("Precision can't be < 0");
+        }
+        if (floatPrecision > MAX_DECIMAL_PRECISION) {
+            throw new IllegalArgumentException("Precision can't be > MAX_DECIMAL_PRECISION (" + MAX_DECIMAL_PRECISION + ")");
+        }
         this.floatPrecision = floatPrecision;
     }
 

diff --git a/src/test/java/org/kdb/inside/brains/view/console/KdbOutputFormatterTest.java b/src/test/java/org/kdb/inside/brains/view/console/KdbOutputFormatterTest.java
@@ -6,13 +6,15 @@
 import org.kdb.inside.brains.view.FormatterOptions;
 import org.kdb.inside.brains.view.KdbOutputFormatter;
 
+import java.math.BigDecimal;
 import java.math.RoundingMode;
 import java.sql.Date;
 import java.sql.Time;
 import java.sql.Timestamp;
 import java.util.UUID;
 
 import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertThrows;
 
 class KdbOutputFormatterTest {
     private FormatterOptions options;
@@ -309,14 +311,20 @@ void nulls() {
     @Test
     void precision() {
         numericalOptions.setScientificNotation(false);
-        numericalOptions.setFloatPrecision(15);
-        assertEquals("24.123456789098764", convert(24.1234567890987654321));
-
-        numericalOptions.setFloatPrecision(2);
-        assertEquals("24.12", convert(24.1234567890987654321));
+        assertThrows(IllegalArgumentException.class, () -> numericalOptions.setFloatPrecision(-1));
+        assertThrows(IllegalArgumentException.class, () -> numericalOptions.setFloatPrecision(NumericalOptions.MAX_DECIMAL_PRECISION + 1));
 
-        numericalOptions.setFloatPrecision(0);
-        assertEquals("24.", convert(24.1234567890987654321));
+        final double d = 1.1234567891234567891;
+        final String s = new BigDecimal(d).toPlainString();
+        numericalOptions.setRoundingMode(RoundingMode.DOWN);
+        for (int i = 0; i <= NumericalOptions.MAX_DECIMAL_PRECISION; i++) {
+            numericalOptions.setFloatPrecision(i);
+            numericalOptions.setScientificNotation(false);
+            assertEquals(s.substring(0, i + 2), convert(d));
+
+            numericalOptions.setScientificNotation(true);
+            assertEquals(s.substring(0, i + 2), convert(d));
+        }
     }
 
     @Test

diff --git a/version.properties b/version.properties
@@ -1 +1 @@
-pluginVersion=5.5.0
+pluginVersion=5.5.1