Binary float writing: Fix double truncation

This commit is contained in:
Lars Mueller 2022-02-28 12:35:15 +01:00
parent 209fb06d6d
commit 5bce5c79cc
2 changed files with 7 additions and 7 deletions

@ -110,24 +110,23 @@ function write_float(write_byte, number, on_write, double)
exponent = exponent - 1 exponent = exponent - 1
end end
local sign_byte = sign + math_floor(exponent / 2) local sign_byte = sign + math_floor(exponent / 2)
mantissa = mantissa * 0x80
local exponent_byte = (exponent % 2) * 0x80 + math_floor(mantissa)
mantissa = mantissa % 1
local mantissa_bytes = {}
-- TODO ensure this check is proper
if double == nil then if double == nil then
double = mantissa % 2^-23 > 0 double = mantissa % 2^-23 ~= 0
end end
if on_write then if on_write then
on_write(double) on_write(double)
end end
mantissa = mantissa * 0x80
local exponent_byte = (exponent % 2) * 0x80 + math_floor(mantissa)
mantissa = mantissa % 1
local mantissa_bytes = {}
local len = double and 6 or 2 local len = double and 6 or 2
for index = 1, len do for index = 1, len do
mantissa = mantissa * 0x100 mantissa = mantissa * 0x100
mantissa_bytes[index] = math_floor(mantissa) mantissa_bytes[index] = math_floor(mantissa)
mantissa = mantissa % 1 mantissa = mantissa % 1
end end
assert(mantissa == 0) assert(mantissa == 0) -- no truncation allowed; round your numbers properly or use auto
for index = len, 1, -1 do for index = len, 1, -1 do
write_byte(mantissa_bytes[index]) write_byte(mantissa_bytes[index])
end end

@ -291,6 +291,7 @@ local function serializer_test(is_json, preserve)
assert_preserves(int) assert_preserves(int)
assert_preserves((random() - 0.5) * 2^random(-20, 20)) assert_preserves((random() - 0.5) * 2^random(-20, 20))
end end
assert_preserves(2.9145637014948988508e-06)
-- Simple tables -- Simple tables
assert_preserves{hello = "world", welt = "hallo"} assert_preserves{hello = "world", welt = "hallo"}
assert_preserves{a = 1, b = "hallo", c = "true"} assert_preserves{a = 1, b = "hallo", c = "true"}