mirror of
https://github.com/appgurueu/modlib.git
synced 2024-11-22 15:23:48 +01:00
Binary float writing: Fix double truncation
This commit is contained in:
parent
209fb06d6d
commit
5bce5c79cc
13
binary.lua
13
binary.lua
@ -110,24 +110,23 @@ function write_float(write_byte, number, on_write, double)
|
|||||||
exponent = exponent - 1
|
exponent = exponent - 1
|
||||||
end
|
end
|
||||||
local sign_byte = sign + math_floor(exponent / 2)
|
local sign_byte = sign + math_floor(exponent / 2)
|
||||||
mantissa = mantissa * 0x80
|
|
||||||
local exponent_byte = (exponent % 2) * 0x80 + math_floor(mantissa)
|
|
||||||
mantissa = mantissa % 1
|
|
||||||
local mantissa_bytes = {}
|
|
||||||
-- TODO ensure this check is proper
|
|
||||||
if double == nil then
|
if double == nil then
|
||||||
double = mantissa % 2^-23 > 0
|
double = mantissa % 2^-23 ~= 0
|
||||||
end
|
end
|
||||||
if on_write then
|
if on_write then
|
||||||
on_write(double)
|
on_write(double)
|
||||||
end
|
end
|
||||||
|
mantissa = mantissa * 0x80
|
||||||
|
local exponent_byte = (exponent % 2) * 0x80 + math_floor(mantissa)
|
||||||
|
mantissa = mantissa % 1
|
||||||
|
local mantissa_bytes = {}
|
||||||
local len = double and 6 or 2
|
local len = double and 6 or 2
|
||||||
for index = 1, len do
|
for index = 1, len do
|
||||||
mantissa = mantissa * 0x100
|
mantissa = mantissa * 0x100
|
||||||
mantissa_bytes[index] = math_floor(mantissa)
|
mantissa_bytes[index] = math_floor(mantissa)
|
||||||
mantissa = mantissa % 1
|
mantissa = mantissa % 1
|
||||||
end
|
end
|
||||||
assert(mantissa == 0)
|
assert(mantissa == 0) -- no truncation allowed; round your numbers properly or use auto
|
||||||
for index = len, 1, -1 do
|
for index = len, 1, -1 do
|
||||||
write_byte(mantissa_bytes[index])
|
write_byte(mantissa_bytes[index])
|
||||||
end
|
end
|
||||||
|
1
test.lua
1
test.lua
@ -291,6 +291,7 @@ local function serializer_test(is_json, preserve)
|
|||||||
assert_preserves(int)
|
assert_preserves(int)
|
||||||
assert_preserves((random() - 0.5) * 2^random(-20, 20))
|
assert_preserves((random() - 0.5) * 2^random(-20, 20))
|
||||||
end
|
end
|
||||||
|
assert_preserves(2.9145637014948988508e-06)
|
||||||
-- Simple tables
|
-- Simple tables
|
||||||
assert_preserves{hello = "world", welt = "hallo"}
|
assert_preserves{hello = "world", welt = "hallo"}
|
||||||
assert_preserves{a = 1, b = "hallo", c = "true"}
|
assert_preserves{a = 1, b = "hallo", c = "true"}
|
||||||
|
Loading…
Reference in New Issue
Block a user