From f7d2b005e1de385bf298b8e0ad6c8ca8301d1b7e Mon Sep 17 00:00:00 2001 From: Lars Mueller Date: Sun, 26 Feb 2023 18:35:43 +0100 Subject: [PATCH] Add highly experimental b3d to glTF conversion --- b3d.lua | 498 +++++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 495 insertions(+), 3 deletions(-) diff --git a/b3d.lua b/b3d.lua index e9af384..6f59628 100644 --- a/b3d.lua +++ b/b3d.lua @@ -1,10 +1,12 @@ -- Localize globals -local assert, error, math, modlib, next, ipairs, pairs, setmetatable, string_char, table - = assert, error, math, modlib, next, ipairs, pairs, setmetatable, string.char, table +local assert, error, math, minetest, modlib, next, ipairs, pairs, setmetatable, string_char, table + = assert, error, math, minetest, modlib, next, ipairs, pairs, setmetatable, string.char, table + +local mat4 = modlib.matrix4 local read_int, read_single = modlib.binary.read_int, modlib.binary.read_single -local write_int, write_single = modlib.binary.write_int, modlib.binary.write_single +local write_int, write_uint, write_single = modlib.binary.write_int, modlib.binary.write_uint, modlib.binary.write_single local fround = modlib.math.fround @@ -534,6 +536,496 @@ function write(self, stream) end end +-- B3D to glTF converter +-- See https://registry.khronos.org/glTF/specs/2.0/glTF-2.0.html +--! Highly experimental; expect bugs! +do + -- glTF constants + local array_buffer = 34962 -- "Buffer containing vertex attributes, such as vertices, texcoords or colors." + local element_array_buffer = 34963 -- "Buffer used for element indices." + local component_type = { + signed_byte = 5120, + unsigned_byte = 5121, + signed_short = 5122, + unsigned_short = 5123, + unsigned_int = 5125, + float = 5126, + } + + -- Coordinate system conversions: + -- "Blitz 3D uses a left-handed system: X+ is to the right. Y+ is up. Z+ is forward." + -- "glTF uses a right-handed coordinate system. glTF defines +Y as up, +Z as forward, and -X as right; + -- the front of a glTF asset faces +Z." + + local function translation_to_gltf(vec) + return {-vec[1], vec[2], vec[3]} -- invert the X-axis + end + + local function quaternion_to_gltf(quat) + -- TODO (!) is this correct? + return {-quat[1], quat[2], quat[3], quat[4]} -- invert the X-axis + end + + -- Convert a color from table format to glTF RGBA list format + local function color_to_gltf(col) + return {col.r, col.g, col.b, col.a} + end + + -- Basic helpers for writing to the buffer, all parameterized in terms of `write_byte` + + local function write_index(write_byte, index) + write_uint(write_byte, index - 1 --[[1-based to 0-based]], 4) + end + + local function write_float(write_byte, float) + assert(-math.huge < float and float < math.huge) + assert(-math.huge < fround(float) and fround(float) < math.huge, ("%.18g got %.18g"):format(float, fround(float))) + write_single(write_byte, fround(float)) + end + + local function write_floats(write_byte, floats, expected_len) + assert(#floats == expected_len) + for i = 1, expected_len do + write_float(write_byte, floats[i]) + end + end + + local function write_vector(write_byte, vec) + return write_floats(write_byte, vec, 3) + end + + local function write_translation(write_byte, vec) + return write_vector(write_byte, translation_to_gltf(vec)) + end + + local function write_quaternion(write_byte, quat) + -- XYZW order is already correct, but we still need to convert left-handed to right-handed + return write_floats(write_byte, quaternion_to_gltf(quat), 4) + end + + function to_gltf(self) + -- Accessor helper: Stores arrays of raw data in a buffer, produces views & accessors. + -- Everything is dumped in the same large buffer. + local buffer_rope = {} -- buffer content (table of strings) + local buffer_views = {} -- glTF buffer views + local accessors = {} -- glTF accessors + local offset = 0 -- current byte offset + local function add_accessor( + type, -- name of the composite type (e.g. SCALAR, VEC3, VEC4, MAT4, ...) + comp_type, -- name of the component type (e.g. float, unsigned_int, ...) + index, -- true / false / nil: whether this is an index (true) or vertex data (false) or neither (nil) + func -- `function(write_byte) ... return count, min, max end` to be called to write to the buffer view; + -- the count of elements written must be returned; min and max may be returned + ) + local bytes_written = 0 + local count, min, max = func(function(byte) + table.insert(buffer_rope, string_char(byte)) + bytes_written = bytes_written + 1 + end) + assert(count) + + -- Add buffer view + table.insert(buffer_views, { + buffer = 0, -- 0-based - there only is one buffer + byteOffset = offset, + byteLength = bytes_written, + target = ((index == true) and element_array_buffer) -- index data + or ((index == false) and array_buffer) -- vertex data + or nil, -- no target hint + }) + table.insert(accessors, { + bufferView = #buffer_views - 1, -- 0-based + byteOffset = 0, -- view has correct offset + componentType = assert(component_type[comp_type]), + type = type, + count = count, + min = min, + max = max, + }) + + offset = offset + bytes_written + return #accessors - 1 -- 0-based index of the accessor + end + + local textures = {} -- glTF textures + local function add_texture(name) + -- TODO (?) add an appropriate sampler + table.insert(textures, {name = name}) + return #textures - 1 -- 0-based texture index + end + for _, tex in ipairs(self.textures) do + -- Assert that all values we don't map properly yet are defaults + -- TODO dig into Blitz3D sources to figure out the meaning of flags & blend + -- TODO (...) deal with flag value of 65536: + -- "The flags field value can conditional an additional flag value of '65536'. + -- This is used to indicate that the texture uses secondary UV values, ala the TextureCoords command." + assert(tex.flags == 1) -- TODO (?) see https://github.com/blitz-research/blitz3d/blob/master/gxruntime/gxcanvas.h#L59 + assert(tex.blend == 2) + -- Assert that the texture isn't transformed + assert(tex.rotation == 0) + assert(tex.pos[1] == 0 and tex.pos[2] == 0) + assert(tex.scale[1] == 1 and tex.scale[2] == 1) + add_texture(tex.file) + end + + -- Map brushes to materials (& textures) + local materials = {} + for i, brush in ipairs(self.brushes) do + -- Assert defaults + -- See https://github.com/blitz-research/blitz3d/blob/6beb288cb5962393684a59a4a44ac11524894939/blitz3d/brush.cpp#L164-L167: + -- 0 = default/replace, 1 = alpha, 2 = multiply, 3 = add + assert(brush.blend == 1) -- (alpha) + -- TODO (...) figure out what these "effects" are and if/how to map them to glTF + assert(brush.fx == 0) + assert(#brush.texture_id <= 1) -- TODO (...) this supports only a single texture per brush for now + local index + if brush.texture_id[1] then + index = brush.texture_id[1] -- 0-based + else + -- Implementations seem to implicitly assume textures for brushes + index = add_texture(brush.name) + end + materials[i] = { + name = brush.name, + alphaMode = "BLEND", + pbrMetallicRoughness = { + baseColorFactor = color_to_gltf(brush.color), + metallicFactor = brush.shininess, -- TODO (?) are these really equivalent? + -- Add texture if there is none + baseColorTexture = { + index = index, + -- `texCoord = 0` is the default already, no need to set it + }, + }, + } + end + + local meshes = {} + local function add_mesh(mesh, weights) + local attributes = {} + + local vertices = mesh.vertices + attributes.POSITION = add_accessor("VEC3", "float", false, function(write_byte) + local inf = math.huge + local min_pos, max_pos = {inf, inf, inf}, {-inf, -inf, -inf} + for _, vertex in ipairs(mesh.vertices) do + local pos = translation_to_gltf(vertex.pos) + write_vector(write_byte, pos) + min_pos = modlib.vector.combine(min_pos, pos, math.min) + max_pos = modlib.vector.combine(max_pos, pos, math.max) + end + return #mesh.vertices, min_pos, max_pos -- vertex accessors MUST provide min & max + end) + + local has_normals = vertices.flags % 2 == 1 -- lowest bit set? + if has_normals then + attributes.NORMAL = add_accessor("VEC3", "float", false, function(write_byte) + for _, vertex in ipairs(mesh.vertices) do + write_translation(write_byte, vertex.normal) + end + return #mesh.vertices + end) + end + + local has_colors = vertices.flags % 4 >= 2 -- second lowest bit set? + if has_colors then + attributes.COLOR_0 = add_accessor("VEC4", "float", false, function(write_byte) + for _, vertex in ipairs(mesh.vertices) do + write_floats(write_byte, color_to_gltf(vertex.color), 4) + end + return #mesh.vertices + end) + end + + if vertices.tex_coord_sets >= 1 then + assert(vertices.tex_coord_set_size == 2) + for tex_coord_set = 1, vertices.tex_coord_sets do + local tcs_id = tex_coord_set - 1 -- 0-based + attributes[("TEXCOORD_%d"):format(tcs_id)] = add_accessor("VEC2", "float", false, function(write_byte) + for _, vertex in ipairs(mesh.vertices) do + write_floats(write_byte, vertex.tex_coords[tex_coord_set], 2) + end + return #mesh.vertices + end) + end + end + + if next(weights) ~= nil then + -- Count (& pack into list) joints influencing vertices, normalize weights + local max_count = 0 + local joint_ids = {} + local normalized_weights = {} + for vertex_id, joint_weights in pairs(weights) do + local total_weight = 0 + local count = 0 + for _, weight in pairs(joint_weights) do + total_weight = total_weight + weight + count = count + 1 + end + joint_ids[vertex_id] = {} + normalized_weights[vertex_id] = {} + for joint, weight in pairs(joint_weights) do + table.insert(joint_ids[vertex_id], joint) + table.insert(normalized_weights[vertex_id], weight / total_weight) + end + max_count = math.max(max_count, count) + end + assert(max_count > 0) -- TODO (?) warning for max_count > 4 + for set_start = 1, max_count, 4 do -- Iterate sets of 4 bones + local set_id = math.floor(set_start / 4) -- 0-based => floor rather than ceil + -- Write the joint IDs + attributes[("JOINTS_%d"):format(set_id)] = add_accessor("VEC4", "unsigned_short", false, function(write_byte) + for vertex_id in ipairs(mesh.vertices) do + for i = set_start, set_start + 3 do + assert(#normalized_weights[vertex_id] == #joint_ids[vertex_id]) + local id = (joint_ids[vertex_id] or {})[i] or 0 + local weight = (normalized_weights[vertex_id] or {})[i] or 0 + if weight == 0 then + id = 0 -- required by the glTF spec + end + write_uint(write_byte, id, 2) + end + end + return #mesh.vertices + end) + -- Write the corresponding weights + attributes[("WEIGHTS_%d"):format(set_id)] = add_accessor("VEC4", "float", false, function(write_byte) + for vertex_id in ipairs(mesh.vertices) do + for i = set_start, set_start + 3 do + local weight = (normalized_weights[vertex_id] or {})[i] or 0 + write_float(write_byte, weight) + end + end + return #mesh.vertices + end) + end + end + + -- Write the indices per triangle set + local primitives = {} + for i, triangle_set in ipairs(mesh.triangle_sets) do + local index_accessor = add_accessor("SCALAR", "unsigned_int", true, function(write_byte) + for _, tri in ipairs(triangle_set.vertex_ids) do + -- Flip winding order due to the coordinate system transformation + -- TODO (!) is this correct? + for j = 3, 1, -1 do + write_index(write_byte, tri[j]) + end + end + return 3 * #triangle_set.vertex_ids + end) + -- Each triangle set is equivalent to one glTF "primitive" + primitives[i] = { + attributes = attributes, + indices = index_accessor, + material = (triangle_set.brush_id or mesh.brush_id) - 1, -- 0-based + -- `mode = 4` (triangles) is the default already, no need to set it + } + end + + table.insert(meshes, {primitives = primitives}) + return #meshes - 1 -- 0-based + end + + -- glTF lists + local nodes = {} + local skins = {} + local samplers = {} + local channels = {} + local function add_node( + node, -- b3d node to add + bind_mat, -- bind matrix of the parent bone (may be `nil` if none) + fps, -- fps of the parent bone (may be `nil` if none) + anim -- shared animation of the parent mesh + ) + table.insert(nodes, false) -- HACK first insert a placeholder to get a fixed ID + local node_id = #nodes - 1 -- 0-indexed <=> before `table.insert`! + + -- Animation (speed)? + fps = node.animation and node.animation.fps or fps + + -- Keyframes? + if node.keys then + -- Convert from a list of keyframes of three overrides to three lists of channels + local targets = { + translation = {output_type = "VEC3", b3d_field = "position", write_value = write_translation}, + scale = {output_type = "VEC3", b3d_field = "scale", write_value = write_vector}, + rotation = {output_type = "VEC4", b3d_field = "rotation", write_value = write_quaternion} + } + for _, keyframe in ipairs(node.keys) do + local frame = keyframe.frame + for _, target in pairs(targets) do + local value = keyframe[target.b3d_field] + if value then + table.insert(target, {frame = frame, value = value}) + end + end + end + for target, keyframes in pairs(targets) do + if #keyframes > 0 then + -- Write input (timestamps) + local input = add_accessor("SCALAR", "float", nil, function(write_byte) + local min, max = math.huge, -math.huge + for _, keyframe in ipairs(keyframes) do + local sec = keyframe.frame / (fps or 60) -- convert frames to seconds; default FPS is 60 + write_float(write_byte, sec) + min, max = math.min(min, sec), math.max(max, sec) + end + return #keyframes, {min}, {max} -- min and max are mandatory + end) + + -- Write output (overrides) + local output = add_accessor(keyframes.output_type, "float", nil, function(write_byte) + for _, keyframe in ipairs(keyframes) do + keyframes.write_value(write_byte, keyframe.value) + end + return #keyframes + end) + + table.insert(samplers, { + input = input, + output = output, + -- interpolation default is already linear, matching b3d + }) + + table.insert(channels, { + sampler = #samplers - 1, -- 0-based + target = { + node = node_id, + path = target, + } + }) + end + end + end + + if node.mesh then + -- Initialize skeletal animation + assert(not anim) + anim = { + weights = {}, + joints = {}, + inv_bind_mats = {}, + } + end + + if node.bone then + local joint_id = #anim.joints + table.insert(anim.joints, node_id) + + -- "To compose the local transformation matrix, TRS properties MUST be converted to matrices and postmultiplied in + -- the T * R * S order; first the scale is applied to the vertices, then the rotation, and then the translation." + local translation = translation_to_gltf(node.position) + local rotation = modlib.quaternion.normalize(quaternion_to_gltf(node.rotation)) + local scale = node.scale + local loc_trans_mat = mat4.scale(scale) + :compose(mat4.rotation(rotation)) + :compose(mat4.translation(translation)) + + -- Compute a proper inverse bind matrix as the inverse of the product of the transformation matrices + -- along the path from the root (the mesh) to the current node (the bone). + -- See e.g. https://stackoverflow.com/questions/17127994/opengl-bone-animation-why-do-i-need-inverse-of-bind-pose-when-working-with-gp + -- https://computergraphics.stackexchange.com/questions/7603/confusion-about-how-inverse-bind-pose-is-actually-calculated-and-used + bind_mat = bind_mat and bind_mat:multiply(loc_trans_mat) or loc_trans_mat + table.insert(anim.inv_bind_mats, bind_mat:inverse()) + + -- Insert into reverse lookup `anim.weights[vertex_id][joint_id] = weight` + -- such that writing the mesh can then write the weights per vertex + for vertex_id, weight in pairs(node.bone) do + if weight > 0 then + anim.weights[vertex_id] = anim.weights[vertex_id] or {} + anim.weights[vertex_id][joint_id] = weight + end + end + end + + local children = {} + for _, child in ipairs(node.children) do + table.insert(children, add_node(child, bind_mat, fps, anim)) + end + local mesh, skin_id + if node.mesh then + mesh = add_mesh(node.mesh, anim.weights) + if anim.joints and anim.joints[1] then + table.insert(skins, { + inverseBindMatrices = add_accessor("MAT4", "float", nil, function(write_byte) + for _, inv_bind_mat in ipairs(anim.inv_bind_mats) do + assert(#inv_bind_mat == 4) + -- glTF uses column-major order (we use row-major order) + for i = 1, 4 do + for j = 1, 4 do + write_float(write_byte, inv_bind_mat[j][i]) + end + end + end + return #anim.inv_bind_mats + end), + joints = anim.joints, + skeleton = anim.joints[1], + }) + skin_id = #skins - 1 -- 0-based + end + end + -- Now replace the placeholder + nodes[node_id + 1 --[[0-based to 1-based]]] = { + name = node.name, + mesh = mesh, + skin = skin_id, + children = children[1] and children, -- glTF does not allow empty lists + translation = translation_to_gltf(node.position), + scale = node.scale, + rotation = quaternion_to_gltf(node.rotation), + } + return node_id -- 0-based + end + + local scene, scenes + if self.node then + scene, scenes = 0, {{nodes = {add_node(self.node)}}} + end + + local buffer_string = table.concat(buffer_rope) + return { + asset = { + generator = "modlib b3d:to_gltf", + version = "2.0" + }, + -- Textures + textures = textures[1] and textures, -- glTF does not allow empty lists + materials = materials, + -- Accessors, buffer views & buffers + accessors = accessors, + bufferViews = buffer_views, + buffers = { + { + byteLength = #buffer_string, + uri = "data:application/octet-stream;base64," + .. assert(minetest.encode_base64, "b3d:to_gltf needs `minetest.encode_base64`!")(buffer_string) + }, + }, + -- Meshes & nodes + meshes = meshes, + nodes = nodes, + -- A scene is not strictly needed but is useful for getting rid of validator warnings & having a proper root defined + scene = scene, + scenes = scenes, + -- Animation + skins = skins, + -- B3D only contains (up to) a single animation + animations = channels[1] and { + { + channels = channels, + samplers = samplers, + }, + }, + } + end +end + +function write_gltf(self, file) + modlib.json:write_file(self:to_gltf(), file) +end + local binary_search_frame = modlib.table.binary_search_comparator(function(a, b) return modlib.table.default_comparator(a, b.frame) end)