From 5bce5c79cc46a3d8fb47b83b73397d44cf09bb41 Mon Sep 17 00:00:00 2001 From: Lars Mueller Date: Mon, 28 Feb 2022 12:35:15 +0100 Subject: [PATCH] Binary float writing: Fix double truncation --- binary.lua | 13 ++++++------- test.lua | 1 + 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/binary.lua b/binary.lua index ea11a35..48502e4 100644 --- a/binary.lua +++ b/binary.lua @@ -110,24 +110,23 @@ function write_float(write_byte, number, on_write, double) exponent = exponent - 1 end local sign_byte = sign + math_floor(exponent / 2) - mantissa = mantissa * 0x80 - local exponent_byte = (exponent % 2) * 0x80 + math_floor(mantissa) - mantissa = mantissa % 1 - local mantissa_bytes = {} - -- TODO ensure this check is proper if double == nil then - double = mantissa % 2^-23 > 0 + double = mantissa % 2^-23 ~= 0 end if on_write then on_write(double) end + mantissa = mantissa * 0x80 + local exponent_byte = (exponent % 2) * 0x80 + math_floor(mantissa) + mantissa = mantissa % 1 + local mantissa_bytes = {} local len = double and 6 or 2 for index = 1, len do mantissa = mantissa * 0x100 mantissa_bytes[index] = math_floor(mantissa) mantissa = mantissa % 1 end - assert(mantissa == 0) + assert(mantissa == 0) -- no truncation allowed; round your numbers properly or use auto for index = len, 1, -1 do write_byte(mantissa_bytes[index]) end diff --git a/test.lua b/test.lua index 4bb2a73..ae3f12c 100644 --- a/test.lua +++ b/test.lua @@ -291,6 +291,7 @@ local function serializer_test(is_json, preserve) assert_preserves(int) assert_preserves((random() - 0.5) * 2^random(-20, 20)) end + assert_preserves(2.9145637014948988508e-06) -- Simple tables assert_preserves{hello = "world", welt = "hallo"} assert_preserves{a = 1, b = "hallo", c = "true"}