Add API to provide descriptions as Minetest hypertext markup

app/blueprints/api/endpoints.py

@@ -36,6 +36,7 @@ from . import bp
 from .auth import is_api_authd
 from .support import error, api_create_vcs_release, api_create_zip_release, api_create_screenshot, \
 	api_order_screenshots, api_edit_package, api_set_cover_image
+from ...utils.minetest_hypertext import html_to_minetest
 
 
 def cors_allowed(f):
@@ -84,6 +85,16 @@ def package(package):
 	return jsonify(package.getAsDictionary(current_app.config["BASE_URL"]))
 
 
+@bp.route("/api/packages/<author>/<name>/hypertext/")
+@is_package_page
+@cors_allowed
+def package_hypertext(package):
+	formspec_version = request.args["formspec_version"]
+	include_images = isYes(request.args.get("include_images", "true"))
+	html = render_markdown(package.desc)
+	return jsonify(html_to_minetest(html, formspec_version, include_images))
+
+
 @bp.route("/api/packages/<author>/<name>/", methods=["PUT"])
 @csrf.exempt
 @is_package_page

app/flatpages/help/api.md

@@ -8,7 +8,7 @@ title: API
 
 ## Responses and Error Handling
 
-If there is an error, the response will be JSON similar to the following with a non-200 status code: 
+If there is an error, the response will be JSON similar to the following with a non-200 status code:
 
 ```json
 {
@@ -26,7 +26,7 @@ often other keys with information. For example:
 {
     "success": true,
     "release": {
-        /* same as returned by a GET */ 
+        /* same as returned by a GET */
     }
 }
 ```
@@ -39,7 +39,7 @@ the number of items is specified using `num`
 
 The response will be a dictionary with the following keys:
 
-* `page`: page number, integer from 1 to max 
+* `page`: page number, integer from 1 to max
 * `per_page`: number of items per page, same as `n`
 * `page_count`: number of pages
 * `total`: total number of results
@@ -55,7 +55,7 @@ Not all endpoints require authentication, but it is done using Bearer tokens:
 
 ```bash
 curl https://content.minetest.net/api/whoami/ \
-    -H "Authorization: Bearer YOURTOKEN" 
+    -H "Authorization: Bearer YOURTOKEN"
 ```
 
 Tokens can be attained by visiting [Settings > API Tokens](/user/tokens/).
@@ -83,7 +83,7 @@ Tokens can be attained by visiting [Settings > API Tokens](/user/tokens/).
         * `tags`: List of [tag](#tags) names.
         * `content_warnings`: List of [content warning](#content-warnings) names.
         * `license`: A [license](#licenses) name.
-        * `media_license`: A [license](#licenses) name.   
+        * `media_license`: A [license](#licenses) name.
         * `long_description`: Long markdown description.
         * `repo`: Git repo URL.
         * `website`: Website URL.
@@ -92,15 +92,27 @@ Tokens can be attained by visiting [Settings > API Tokens](/user/tokens/).
         * `video_url`: URL to a video.
         * `donate_url`: URL to a donation page.
         * `game_support`: Array of game support information objects. Not currently documented, as subject to change.
+* GET `/api/packages/<author>/<name>/hypertext/`
+    * Converts the long description to [Minetest Markup Language](https://github.com/minetest/minetest/blob/master/doc/lua_api.md#markup-language)
+      to be used in a `hypertext` formspec element.
+    * Query arguments:
+        * `formspec_version`: Required, maximum supported formspec version.
+        * `include_images`: Optional, defaults to true.
+    * Returns JSON dictionary with following key:
+        * `head`: markup for suggested styling and custom tags, prepend to the body before displaying.
+        * `body`: markup for long description.
+        * `links`: dictionary of anchor name to link URL.
+        * `images`: dictionary of img name to image URL
+        * `image_tooltips`: dictionary of img name to tooltip text.
 * GET `/api/packages/<username>/<name>/dependencies/`
-    * Returns dependencies, with suggested candidates 
+    * Returns dependencies, with suggested candidates
     * If query argument `only_hard` is present, only hard deps will be returned.
 * GET `/api/dependencies/`
     * Returns `provides` and raw dependencies for all packages.
     * Supports [Package Queries](#package-queries)
     * [Paginated result](#paginated-results), max 300 results per page
     * Each item in `items` will be a dictionary with the following keys:
-        * `type`: One of `GAME`, `MOD`, `TXP`. 
+        * `type`: One of `GAME`, `MOD`, `TXP`.
         * `author`: Username of the package author.
         * `name`: Package name.
         * `provides`: List of technical mod names inside the package.
@@ -142,7 +154,7 @@ Examples:
 curl -X PUT https://content.minetest.net/api/packages/username/name/ \
     -H "Authorization: Bearer YOURTOKEN" -H "Content-Type: application/json" \
     -d '{ "title": "Foo bar", "tags": ["pvp", "survival"], "license": "MIT" }'
-    
+
 # Remove website URL
 curl -X PUT https://content.minetest.net/api/packages/username/name/ \
     -H "Authorization: Bearer YOURTOKEN" -H "Content-Type: application/json" \
@@ -161,7 +173,7 @@ Supported query parameters:
 * `q`:  Query string.
 * `author`:  Filter by author.
 * `tag`:  Filter by tags.
-* `game`: Filter by [Game Support](/help/game_support/), ex: `Wuzzy/mineclone2`. (experimental, doesn't show items that support every game currently). 
+* `game`: Filter by [Game Support](/help/game_support/), ex: `Wuzzy/mineclone2`. (experimental, doesn't show items that support every game currently).
 * `random`:  When present, enable random ordering and ignore `sort`.
 * `limit`:  Return at most `limit` packages.
 * `hide`:  Hide content based on [Content Flags](/help/content_flags/).
@@ -171,12 +183,12 @@ Supported query parameters:
 * `engine_version`:  Only show packages supported by this Minetest engine version, eg: `5.3.0`.
 * `fmt`:  How the response is formatted.
     * `keys`:  author/name only.
-    * `short`:  stuff needed for the Minetest client. 
+    * `short`:  stuff needed for the Minetest client.
 
 
 ### Releases
 
-* GET `/api/releases/` (List)   
+* GET `/api/releases/` (List)
     * Limited to 30 most recent releases.
     * Optional arguments:
         * `author`: Filter by author
@@ -204,14 +216,14 @@ Supported query parameters:
     * For Git release creation:
         * `method`: must be `git`.
         * `ref`: (Optional) git reference, eg: `master`.
-    * For zip upload release creation: 
+    * For zip upload release creation:
         * `file`: multipart file to upload, like `<input type="file" name="file">`.
         * `commit`: (Optional) Source Git commit hash, for informational purposes.
     * You can set min and max Minetest Versions [using the content's .conf file](/help/package_config/).
 * DELETE `/api/packages/<username>/<name>/releases/<id>/` (Delete)
     * Requires authentication.
     * Deletes release.
-  
+
 Examples:
 
 ```bash
@@ -232,7 +244,7 @@ curl -X POST https://content.minetest.net/api/packages/username/name/releases/ne
 
 # Delete release
 curl -X DELETE https://content.minetest.net/api/packages/username/name/releases/3/ \
-    -H "Authorization: Bearer YOURTOKEN" 
+    -H "Authorization: Bearer YOURTOKEN"
 ```
 
 
@@ -275,7 +287,7 @@ Examples:
 curl -X POST https://content.minetest.net/api/packages/username/name/screenshots/new/ \
     -H "Authorization: Bearer YOURTOKEN" \
     -F title="My Release" -F file=@path/to/screnshot.png
-    
+
 # Create screenshot and set it as the cover image
 curl -X POST https://content.minetest.net/api/packages/username/name/screenshots/new/ \
     -H "Authorization: Bearer YOURTOKEN" \
@@ -283,13 +295,13 @@ curl -X POST https://content.minetest.net/api/packages/username/name/screenshots
 
 # Delete screenshot
 curl -X DELETE https://content.minetest.net/api/packages/username/name/screenshots/3/ \
-    -H "Authorization: Bearer YOURTOKEN" 
+    -H "Authorization: Bearer YOURTOKEN"
-    
+
 # Reorder screenshots
 curl -X POST https://content.minetest.net/api/packages/username/name/screenshots/order/ \
     -H "Authorization: Bearer YOURTOKEN" -H "Content-Type: application/json" \
     -d "[13, 2, 5, 7]"
-    
+
 # Set cover image
 curl -X POST https://content.minetest.net/api/packages/username/name/screenshots/cover-image/ \
     -H "Authorization: Bearer YOURTOKEN" -H "Content-Type: application/json" \
@@ -302,7 +314,7 @@ curl -X POST https://content.minetest.net/api/packages/username/name/screenshots
 * GET `/api/packages/<username>/<name>/reviews/` (List)
     * Returns array of review dictionaries with keys:
         * `user`: dictionary with `display_name` and `username`.
-        * `title`: review title 
+        * `title`: review title
         * `comment`: the text
         * `rating`: 1 for negative, 3 for neutral, 5 for positive
         * `is_positive`: boolean
@@ -326,16 +338,16 @@ Example:
 ```json
 [
   {
-    "comment": "This is a really good mod!", 
+    "comment": "This is a really good mod!",
-    "created_at": "2021-11-24T16:18:33.764084", 
+    "created_at": "2021-11-24T16:18:33.764084",
-    "is_positive": true, 
+    "is_positive": true,
-    "title": "Really good", 
+    "title": "Really good",
     "user": {
-      "display_name": "rubenwardy", 
+      "display_name": "rubenwardy",
       "username": "rubenwardy"
-    }, 
+    },
     "votes": {
-      "helpful": 0, 
+      "helpful": 0,
       "unhelpful": 0
     }
   }
@@ -403,7 +415,7 @@ Supported query parameters:
     * `description`:  tag description or null.
     * `is_protected`: boolean, whether the tag is protected (can only be set by Editors in the web interface).
     * `views`: number of views of this tag.
-    
+
 ### Content Warnings
 
 * GET `/api/content_warnings/` ([View](/api/content_warnings/)):  List of:

app/tests/unit/test_minetest_hypertext.py

@@ -0,0 +1,95 @@
+from app.utils.minetest_hypertext import html_to_minetest
+
+
+conquer_html = """
+<p>
+Welcome to <b>Conquer</b>, a mod that adds RTS gameplay. It allows players to start
+Conquer sub-games, where they can place buildings, train units, and
+fight other players.
+</p>
+
+<h2>Starting or joining a session</h2>
+
+<p>
+You can list running sessions by typing:
+</p>
+
+<pre><code>/conquer list
+new line
+
+another new line</code></pre>
+
+<p>
+You'll switch into Conquer playing mode, where you will be given buildings that you can place.
+You'll need to place a keep, which you must protect at all costs.
+</p>
+
+<p>
+You may leave a game and return to normal playing mode at anytime by typing:
+</p>
+
+<h2>Conquer GUI</h2>
+
+<p>
+The Conquer GUI is the central place for monitoring your kingdom.
+Once in a session, you can view it by pressing the inventory key (I),
+or by punching/right-clicking the keep node.
+</p>
+"""
+
+
+conquer_expected = """
+Welcome to <b>Conquer</b>, a mod that adds RTS gameplay. It allows players to start Conquer sub-games, where they can place buildings, train units, and fight other players.
+
+<big>Starting or joining a session</big>
+You can list running sessions by typing:
+<code>/conquer list
+new line
+
+another new line</code>
+You'll switch into Conquer playing mode, where you will be given buildings that you can place. You'll need to place a keep, which you must protect at all costs.
+You may leave a game and return to normal playing mode at anytime by typing:
+
+<big>Conquer GUI</big>
+The Conquer GUI is the central place for monitoring your kingdom. Once in a session, you can view it by pressing the inventory key (I), or by punching/right-clicking the keep node.
+"""
+
+
+def test_conquer():
+	assert html_to_minetest(conquer_html)["body"].strip() == conquer_expected.strip()
+
+
+def test_images():
+	html = """
+		<img src="/path/to/img.png">
+	"""
+
+	expected = "<img name=image_0 width=128 height=128>"
+	result = html_to_minetest(html)
+	assert result["body"].strip() == expected.strip()
+	assert len(result["images"]) == 1
+	assert result["images"]["image_0"] == "/path/to/img.png"
+
+
+def test_bullets():
+	html = """
+		<ul>
+			<li>One</li>
+			<li>two three</li>
+			<li>four</li>
+		</ul>
+	"""
+
+	expected = "• One\n• two three\n• four\n"
+	result = html_to_minetest(html)
+	assert result["body"].strip() == expected.strip()
+
+
+def test_inline():
+	html = """
+		<b>One <i>two</i> three</b>
+	"""
+
+	expected = "<b>One <i>two</i> three</b>"
+	result = html_to_minetest(html)
+	assert result["body"].strip() == expected.strip()

app/utils/minetest_hypertext.py

@@ -0,0 +1,177 @@
+from html.parser import HTMLParser
+import re
+import sys
+
+
+def normalize_whitespace(x):
+	return re.sub(r"\s+", " ", x)
+
+
+assert normalize_whitespace(" one  three\nfour\n\n") == " one three four "
+
+
+# Styles and custom tags
+HEAD = normalize_whitespace("""
+	<tag name=code color=#7bf font=mono>
+	<tag name=action color=#77f hovercolor=#aaf>
+""").strip()
+
+
+def get_attributes(attrs):
+	retval = {}
+	for attr in attrs:
+		retval[attr[0]] = attr[1]
+	return retval
+
+
+class MinetestHTMLParser(HTMLParser):
+	def __init__(self, include_images):
+		super().__init__()
+		self.include_images = include_images
+
+		self.text_buffer = ""
+		self.has_line_started = False
+		self.links = {}
+		self.images = {}
+		self.image_tooltips = {}
+		self.is_preserving = False
+		self.remove_until = None
+
+	def handle_starttag(self, tag, attrs):
+		if self.is_preserving or self.remove_until:
+			return
+
+		print("OPEN", tag, file=sys.stderr)
+
+		self.has_line_started = True
+		if tag == "p":
+			self.has_line_started = False
+		elif tag == "pre":
+			self.text_buffer += "<code>"
+			self.is_preserving = True
+			self.has_line_started = False
+		elif tag == "table":
+			# Tables are currently unsupported and removed
+			self.remove_until = "table"
+			self.text_buffer += "<i>(table removed)</i>\n"
+		elif tag == "br":
+			self.text_buffer += "\n"
+			self.has_line_started = False
+		elif tag == "h1" or tag == "h2":
+			self.text_buffer += "\n<big>"
+		elif tag == "h3" or tag == "h4" or tag == "h5":
+			self.text_buffer += "\n<b>"
+		elif tag == "a":
+			for attr in attrs:
+				if attr[0] == "href":
+					name = f"link_{len(self.links)}"
+					self.links[name] = attr[1]
+					self.text_buffer += f"<action name={name}><u>"
+					break
+			else:
+				self.text_buffer += "<action><u>"
+		elif tag == "img":
+			attr_by_value = get_attributes(attrs)
+			if "src" in attr_by_value and self.include_images:
+				name = f"image_{len(self.images)}"
+				self.images[name] = attr_by_value["src"]
+				width = attr_by_value.get("width", 128)
+				height = attr_by_value.get("height", 128)
+				self.text_buffer += f"<img name={name} width={width} height={height}>"
+
+				if "alt" in attr_by_value:
+					self.image_tooltips[name] = attr_by_value["alt"]
+		elif tag == "b" or tag == "strong":
+			self.text_buffer += "<b>"
+		elif tag == "i" or tag == "em":
+			self.text_buffer += "<i>"
+		elif tag == "u":
+			self.text_buffer += "<u>"
+		elif tag == "li":
+			self.has_line_started = False
+			self.text_buffer += "• "
+		elif tag == "code":
+			self.text_buffer += "<code>"
+		elif tag == "span" or tag == "ul":
+			pass
+		else:
+			print("UNKNOWN TAG ", tag, attrs, file=sys.stderr)
+
+	def handle_endtag(self, tag):
+		if self.remove_until:
+			if self.remove_until == tag:
+				self.remove_until = None
+			return
+
+		print("CLOSE", tag, file=sys.stderr)
+
+		if tag == "pre":
+			self.text_buffer = self.text_buffer.rstrip()
+			self.text_buffer += "</code>\n"
+			self.is_preserving = False
+			self.has_line_started = False
+		elif self.is_preserving:
+			return
+		elif tag == "p":
+			self.text_buffer = self.text_buffer.rstrip()
+			self.text_buffer += "\n"
+			self.has_line_started = False
+		elif tag == "h1" or tag == "h2":
+			self.text_buffer += "</big>\n"
+			self.has_line_started = False
+		elif tag == "h3" or tag == "h4" or tag == "h5":
+			self.text_buffer += "</b>\n"
+			self.has_line_started = False
+		elif tag == "a":
+			self.text_buffer += "</u></action>"
+		elif tag == "code":
+			self.text_buffer += "</code>"
+		elif tag == "b" or tag == "strong":
+			self.text_buffer += "</b>"
+		elif tag == "i" or tag == "em":
+			self.text_buffer += "</i>"
+		elif tag == "u":
+			self.text_buffer += "</u>"
+		elif tag == "li":
+			self.text_buffer += "\n"
+		# else:
+		# 	print("END", tag, file=sys.stderr)
+
+	def handle_data(self, data):
+		print(f"DATA \"{data}\"", file=sys.stderr)
+		if self.remove_until:
+			return
+
+		if not self.is_preserving:
+			data = normalize_whitespace(data)
+			if not self.has_line_started:
+				data = data.lstrip()
+
+		self.text_buffer += data
+		self.has_line_started = True
+
+	def handle_entityref(self, name):
+		to_value = {
+			"lt": "\\<",
+			"gr": "\\>",
+			"amp": "&",
+			"quot": "\"",
+			"apos": "'",
+		}
+
+		if name in to_value:
+			self.text_buffer += to_value[name]
+		else:
+			self.text_buffer += f"&{name};"
+
+
+def html_to_minetest(html, formspec_version=6, include_images=True):
+	parser = MinetestHTMLParser(include_images)
+	parser.feed(html)
+	return {
+		"head": HEAD,
+		"body": parser.text_buffer.strip() + "\n\n",
+		"links": parser.links,
+		"images":  parser.images,
+		"image_tooltips": parser.image_tooltips,
+	}