modlib/utf8.lua

102 lines
3.1 KiB
Lua
Raw Permalink Normal View History

2022-10-01 19:31:27 +02:00
local assert, error, select, string_char, table_concat
= assert, error, select, string.char, table.concat
2022-09-09 14:17:18 +02:00
local utf8 = {}
-- Overly permissive pattern that greedily matches a single UTF-8 codepoint
utf8.charpattern = "[%z-\127\194-\253][\128-\191]*"
function utf8.is_valid_codepoint(codepoint)
-- Must be in bounds & must not be a surrogate
return codepoint <= 0x10FFFF and (codepoint < 0xD800 or codepoint > 0xDFFF)
end
2022-10-01 19:31:27 +02:00
local function utf8_bytes(codepoint)
if codepoint <= 0x007F then
return codepoint
end if codepoint <= 0x7FF then
local payload_2 = codepoint % 0x40
codepoint = (codepoint - payload_2) / 0x40
return 0xC0 + codepoint, 0x80 + payload_2
end if codepoint <= 0xFFFF then
local payload_3 = codepoint % 0x40
codepoint = (codepoint - payload_3) / 0x40
local payload_2 = codepoint % 0x40
codepoint = (codepoint - payload_2) / 0x40
return 0xE0 + codepoint, 0x80 + payload_2, 0x80 + payload_3
end if codepoint <= 0x10FFFF then
local payload_4 = codepoint % 0x40
codepoint = (codepoint - payload_4) / 0x40
local payload_3 = codepoint % 0x40
codepoint = (codepoint - payload_3) / 0x40
local payload_2 = codepoint % 0x40
codepoint = (codepoint - payload_2) / 0x40
return 0xF0 + codepoint, 0x80 + payload_2, 0x80 + payload_3, 0x80 + payload_4
end error"codepoint out of range"
end
function utf8.char(...)
local n_args = select("#", ...)
if n_args == 0 then
return
end if n_args == 1 then
return string_char(utf8_bytes(...))
end
local chars = {}
for i = 1, n_args do
2022-10-01 19:31:27 +02:00
chars[i] = string_char(utf8_bytes(select(i, ...)))
end
return table_concat(chars)
2022-09-09 14:17:18 +02:00
end
local function utf8_next_codepoint(str, i)
local first_byte = str:byte(i)
if first_byte < 0x80 then
return i + 1, first_byte
end
local len, head_bits
if first_byte >= 0xC0 and first_byte <= 0xDF then -- 110_00000 to 110_11111
len, head_bits = 2, first_byte % 0x20 -- last 5 bits
elseif first_byte >= 0xE0 and first_byte <= 0xEF then -- 1110_0000 to 1110_1111
len, head_bits = 3, first_byte % 0x10 -- last 4 bits
elseif first_byte >= 0xF0 and first_byte <= 0xF7 then -- 11110_000 to 11110_111
len, head_bits = 4, first_byte % 0x8 -- last 3 bits
else error"invalid UTF-8" end
local codepoint = 0
local pow = 1
for j = i + len - 1, i + 1, -1 do
local byte = assert(str:byte(j), "invalid UTF-8")
local val_bits = byte % 0x40 -- extract last 6 bits xxxxxx from 10xxxxxx
assert(byte - val_bits == 0x80) -- assert that first two bits are 10
codepoint = codepoint + val_bits * pow
pow = pow * 0x40
end
return i + len, codepoint + head_bits * pow
end
function utf8.codepoint(str, i, j)
i, j = i or 1, j or #str
if i > j then return end
local codepoint
i, codepoint = utf8_next_codepoint(str, i)
assert(i - j <= 1, "invalid UTF-8")
return codepoint, utf8.codepoint(str, i)
end
-- Iterator to loop over the UTF-8 characters as `index, codepoint`
function utf8.codes(text, i)
i = i or 1
return function()
if i > #text then
return
end
local prev_index = i
local codepoint
i, codepoint = utf8_next_codepoint(text, i)
return prev_index, codepoint
end
end
2022-09-09 14:17:18 +02:00
return utf8