docs: massively improve image conversion system

more formats, memoizing, custom <picture> element generation, async 
image conversion to optimise CPU usage, and more!

Too bad that the imagemagick apt package doesn't natively support JPEG 
XL just yet (Imagemagick 7 does though), but that's an easy fix on our 
end once they add support :D

It looks like we need to implement our own alternative to the serve 
command though :-/
This commit is contained in:
Starbeamrainbowlabs 2021-10-17 03:00:24 +01:00
parent 2ae241aee5
commit 6eaa3799d3
No known key found for this signature in database
GPG Key ID: 1BE5172E637709C2
7 changed files with 541 additions and 1449 deletions

@ -1,12 +1,14 @@
"use strict";
const os = require("os");
const fs = require("fs");
const path = require("path");
const debug = require("debug");
const htmlentities = require("html-entities");
const phin = require("phin");
const Image = require("@11ty/eleventy-img");
const HTMLPicture = require("./lib/HTMLPicture.js");
var nextid = 0;
@ -16,46 +18,22 @@ const image_filename_format = (_id, src, width, format, _options) => {
return `${name}-${width}w.${format}`;
};
function image_metadata_log(metadata, source) {
for(let format in metadata) {
for(let img of metadata[format]) {
console.log(`${source.padEnd(10)} ${format.padEnd(5)} ${`${img.width}x${img.height}`.padEnd(10)} ${img.outputPath}`);
}
}
}
async function shortcode_image(src, alt, classes = "") {
let metadata = await Image(src, {
widths: [300, null],
formats: ["avif", "jpeg"],
outputDir: `./_site/img/`,
filenameFormat: image_filename_format
});
image_metadata_log(metadata, `IMAGE`);
async function shortcode_image(src, alt) {
let imageAttributes = {
class: classes,
alt: htmlentities.encode(alt),
sizes: Object.values(metadata)[0].map((el) => `${el.width}w`).join(" "),
loading: "lazy",
decoding: "async",
};
// You bet we throw an error on missing alt in `imageAttributes` (alt="" works okay)
return Image.generateHTML(metadata, imageAttributes);
return HTMLPicture(
src, alt,
`./_site/img`, `/img`
);
}
async function shortcode_image_url(src) {
let metadata = await Image(src, {
widths: [ null ],
formats: [ "jpeg" ],
outputDir: `./_site/img/`,
filenameFormat: image_filename_format
});
image_metadata_log(metadata, `IMAGE_URL`);
const src_parsed = path.parse(src);
const target = path.join(`./_site/img`, src_parsed.base);
if(!fs.existsSync(path.dirname(target)))
await fs.promises.mkdir(target_dir, { recursive: true });
await fs.promises.copyFile(src, target);
let data = metadata.jpeg[metadata.jpeg.length - 1];
return data.url;
return path.join(`/img`, src_parsed.base);
}
async function shortcode_image_urlpass(src) {
@ -82,14 +60,14 @@ async function shortcode_gallerybox(content, src, idthis, idprev, idnext) {
}
async function fetch(url) {
let package = JSON.parse(await fs.promises.readFile(
const pkg_obj = JSON.parse(await fs.promises.readFile(
path.join(__dirname, "package.json"), "utf8"
));
return (await phin({
url,
headers: {
"user-agent": `WorldEditAdditionsStaticBuilder/${package.version} (Node.js/${process.version}; ${os.platform()} ${os.arch()}) eleventy/${package.devDependencies["@11ty/eleventy"].replace(/\^/, "")}`
"user-agent": `WorldEditAdditionsStaticBuilder/${pkg_obj.version} (Node.js/${process.version}; ${os.platform()} ${os.arch()}) eleventy/${pkg_obj.devDependencies["@11ty/eleventy"].replace(/\^/, "")}`
},
followRedirects: true,
parse: "string"

@ -1,5 +1,12 @@
"use strict";
const fs = require("fs");
const path = require("path");
const columnify = require("columnify");
const htmlentities = require("html-entities");
const a = require("./lib/Ansi.js");
const parse_sections = require("./lib/parse_sections.js");
let { sections, categories } = parse_sections(fs.readFileSync(
@ -15,11 +22,20 @@ sections = sections.slice(1).sort((a, b) => a.title.replace(/^\/+/g, "").localeC
console.log(`REFERENCE SECTION TITLES`)
console.log(sections
.map(s => [s.category, s.title].join(`\t`)).join(`\n`));
console.log(columnify(sections.map(s => { return {
category: `${a.hicol}${a.fyellow}${s.category}${a.reset}`,
command: `${a.hicol}${a.fmagenta}${htmlentities.decode(s.title)}${a.reset}`
} })));
// console.log(sections
// .map(s => `${a.fyellow}${a.hicol}${s.category}${a.reset}\t${a.fmagenta}${a.hicol}${s.title}${a.reset}`).join(`\n`));
console.log(`************************`);
console.log(`REFERENCE SECTION COLOURS`, categories);
console.log(`REFERENCE SECTION COLOURS`);
console.log(columnify(Array.from(categories).map(el => { return {
category: el[0],
colour: el[1]
} })));
module.exports = {
layout: "theme.njk",
title: "Reference",

88
.docs/lib/Ansi.js Normal file

@ -0,0 +1,88 @@
"use strict";
/**
* Generates various VT100 ANSI escape sequences.
* Ported from C#.
* @licence MPL-2.0 <https://www.mozilla.org/en-US/MPL/2.0/>
* @source https://gist.github.com/a4edd3204a03f4eedb79785751efb0f3#file-ansi-cs
* @author Starbeamrainbowlabs
* GitHub: @sbrl | Twitter: @SBRLabs | Reddit: u/Starbeamrainbowlabs
***** Changelog *****
* 27th March 2019:
* - Initial public release
* 9th March 2020:
* - Add Italics (\u001b[3m])
* - Export a new instance of it by default (makes it so that there's 1 global instance)
* 5th September 2020:
* - Add support for NO_COLOR environment variable <https://no-color.org/>
*/
class Ansi {
constructor() {
/**
* Whether we should *actually* emit ANSI escape codes or not.
* Useful when we want to output to a log file, for example
* @type {Boolean}
*/
this.enabled = true;
this.escape_codes();
}
escape_codes() {
if(typeof process !== "undefined" && typeof process.env.NO_COLOR == "string") {
this.enabled = false;
return;
}
// Solution on how to output ANSI escape codes in C# from here:
// https://www.jerriepelser.com/blog/using-ansi-color-codes-in-net-console-apps
this.reset = this.enabled ? "\u001b[0m" : "";
this.hicol = this.enabled ? "\u001b[1m" : "";
this.locol = this.enabled ? "\u001b[2m" : "";
this.italics = this.enabled ? "\u001b[3m" : "";
this.underline = this.enabled ? "\u001b[4m" : "";
this.inverse = this.enabled ? "\u001b[7m" : "";
this.fblack = this.enabled ? "\u001b[30m" : "";
this.fred = this.enabled ? "\u001b[31m" : "";
this.fgreen = this.enabled ? "\u001b[32m" : "";
this.fyellow = this.enabled ? "\u001b[33m" : "";
this.fblue = this.enabled ? "\u001b[34m" : "";
this.fmagenta = this.enabled ? "\u001b[35m" : "";
this.fcyan = this.enabled ? "\u001b[36m" : "";
this.fwhite = this.enabled ? "\u001b[37m" : "";
this.bblack = this.enabled ? "\u001b[40m" : "";
this.bred = this.enabled ? "\u001b[41m" : "";
this.bgreen = this.enabled ? "\u001b[42m" : "";
this.byellow = this.enabled ? "\u001b[43m" : "";
this.bblue = this.enabled ? "\u001b[44m" : "";
this.bmagenta = this.enabled ? "\u001b[45m" : "";
this.bcyan = this.enabled ? "\u001b[46m" : "";
this.bwhite = this.enabled ? "\u001b[47m" : "";
}
// Thanks to http://ascii-table.com/ansi-escape-sequences.php for the following ANSI escape sequences
up(lines = 1) {
return this.enabled ? `\u001b[${lines}A` : "";
}
down(lines = 1) {
return this.enabled ? `\u001b[${lines}B` : "";
}
right(lines = 1) {
return this.enabled ? `\u001b[${lines}C` : "";
}
left(lines = 1) {
return this.enabled ? `\u001b[${lines}D` : "";
}
jump_to(x, y) {
return this.enabled ? `\u001b[${y};${x}H` : "";
}
cursor_pos_save() {
return this.enabled ? `\u001b[s` : "";
}
cursor_pos_restore() {
return this.enabled ? `\u001b[u` : "";
}
}
module.exports = new Ansi();

154
.docs/lib/HTMLPicture.js Normal file

@ -0,0 +1,154 @@
"use strict";
const os = require(`os`);
const fs = require("fs");
const path = require("path");
const pretty_ms = require("pretty-ms");
const debug = require("debug")("image");
const imagickal = require("imagickal");
const htmlentities = require("html-entities");
const a = require("./Ansi.js");
function calculate_size(width, height, size_spec) {
if(size_spec.indexOf("%") > -1) {
// It's a percentage
const multiplier = parseInt(size_spec.replace(/%/, ""), 10) / 100;
return {
width: Math.ceil(width * multiplier),
height: Math.ceil(height * multiplier)
};
}
else {
// It's an absolute image width
const new_width = parseInt(size_spec, 10);
return {
width: new_width,
height: Math.ceil(new_width/width * height)
};
}
}
// Main task list - we make sure it completes before exiting.
var queue = null;
var pMemoize = null;
async function make_queue() {
// 1: Setup task queue
const PQueue = (await import("p-queue")).default;
let concurrency = os.cpus().length;
if(process.env["MAX_CONCURRENT"])
concurrency = parseInt(process.env["MAX_CONCURRENT"], 10);
debug(`Image conversion queue concurrency: `, concurrency);
queue = new PQueue({ concurrency });
queue.on("idle", () => console.log(`IMAGE ${a.fcyan}all conversions complete${a.reset}`));
process.on("exit", async () => {
debug(`Waiting for image conversions to finish...`);
await queue.onEmpty();
debug(`All image conversions complete.`);
});
}
async function srcset(source_image, target_dir, urlpath, format = "__AUTO__", sizes = [ "25%", "50%", "100%" ], quality = 95, strip = true) {
if(queue === null) await make_queue();
const source_parsed = path.parse(source_image);
// ext contains the dot . already
const target_format = format == "__AUTO__" ? source_parsed.ext.replace(/\./g, "") : format;
const source_size = await imagickal.dimensions(source_image);
debug(`SOURCE_SIZE`, source_size, `TARGET_FORMAT`, target_format);
let setitems = await Promise.all(sizes.map(async (size) => {
let target_filename = `${source_parsed.name}_${size}.${target_format}`;
let target_current = path.join(
target_dir,
target_filename
);
queue.add(async () => {
const start = new Date();
await imagickal.transform(source_image, target_current, {
resize: { width: size },
quality,
strip
});
console.log(`IMAGE\t${a.fcyan}${queue.size}/${queue.pending} tasks${a.reset}\t${a.fyellow}${pretty_ms(new Date() - start)}${a.reset}\t${a.fgreen}${target_current}${a.reset}`);
});
// const size_target = await imagickal.dimensions(target_current);
const predict = calculate_size(source_size.width, source_size.height, size);
// debug(`size spec:`, size, `size predicted: ${predict.width}x${predict.height} actual: ${size_target.width}x${size_target.height}`);
return `${path.resolve(urlpath, target_filename)} ${predict.width}w`;
}));
return setitems.join(", ");
}
/**
* Generates a string of HTML for a <picture> element, converting images to the specified formats in the process.
* @param {string} source_image The filepath to the source image.
* @param {string} alt The alt (alternative) text. Automatically run though htmlentities.
* @param {string} target_dir The target directory to save converted images to.
* @param {string} urlpath The path to the aforementionoed target directory as a URL. Image paths in the HTML will be prefixed with this value.
* @param {string} [formats="__AUTO__"] A list of formats to convert the source image to. Defaults to automatically determining the most optimal formats based on the input format. [must be lowercase]
* @param {Array} [sizes=["25%","50%", "100%" ]] The sizes, as imagemagick size specs, to convert the source image to.
* @param {Number} [quality=95] The quality value to use when converting images.
* @param {Boolean} [strip=true] Whether to strip all metadata from images when converting them [saves some space]
* @return {Promise<string>} A Promise that returns a generated string of HTML.
*/
async function picture(source_image, alt, target_dir, urlpath, formats = "__AUTO__", sizes = [ "25%", "50%", "100%" ], quality = 95, strip = true) {
const source_parsed = path.parse(source_image);
const source_format = source_parsed.ext.toLowerCase().replace(".", "");
if(formats == "__AUTO__") {
switch(source_format) {
case "png":
case "gif": // * shudder *
case "bmp":
formats = [ "png" ];
break;
default:
// jxl = JPEG XL <https://jpegxl.info/> - not currently supported by the old version of imagemagick shipped via apt :-/
// Imagemagick v7+ does support it, but isn't shipped yet :-(
formats = [ "jpeg", "webp", "avif", /*"jxl"*/ ];
break;
}
}
const target_original = path.join(target_dir, source_parsed.base);
await fs.promises.copyFile(source_image, target_original);
const sources = await Promise.all(formats.map(async (format) => {
debug(`${format} ${source_image}`);
return {
mime: `image/${format}`,
srcset: await srcset(
source_image,
target_dir, urlpath,
format, sizes,
quality, strip
)
};
}));
let result = `<picture>\n\t`;
result += sources.map(source => `<source srcset="${source.srcset}" type="${source.mime}" />`).join(`\n\t`);
result += `\n\t<img loading="lazy" decoding="async" src="${urlpath}/${source_parsed.base}" alt="${htmlentities.encode(alt)}" />\n`;
result += `</picture>\n`
return result;
}
var picture_memoize = null;
async function setup_memoize() {
const pMemoize = (await import("p-memoize")).default;
picture_memoize = pMemoize(picture);
}
module.exports = async function(...args) {
if(picture_memoize === null) await setup_memoize();
return await picture_memoize(...args);
};

@ -1,3 +1,5 @@
"use strict";
const crypto = require("crypto");
const htmlentities = require("html-entities");
@ -6,7 +8,6 @@ const markdown = require("markdown-it")({
});
const chroma = require("chroma-js");
const markdown_prism = require("markdown-it-prism");
markdown.use(markdown_prism, {
init: (Prism) => {

1658
.docs/package-lock.json generated

File diff suppressed because it is too large Load Diff

@ -21,10 +21,15 @@
"homepage": "https://github.com/sbrl/Minetest-WorldEditAdditions#readme",
"devDependencies": {
"@11ty/eleventy": "^0.12.1",
"@11ty/eleventy-img": "^0.10.0",
"chroma-js": "^2.1.2",
"columnify": "^1.5.4",
"debug": "^4.3.2",
"imagickal": "^5.0.1",
"markdown-it-prism": "^2.2.1",
"phin": "^3.6.0"
"p-memoize": "^6.0.1",
"p-queue": "^7.1.0",
"phin": "^3.6.0",
"pretty-ms": "^7.0.1"
},
"dependencies": {
"html-entities": "^2.3.2"