From 0a06e41497cb90fc51bc08c7d4173fddae6140b8 Mon Sep 17 00:00:00 2001 From: rubenwardy Date: Wed, 19 Apr 2023 18:27:28 +0100 Subject: [PATCH] Add API to provide descriptions as Minetest hypertext markup --- app/blueprints/api/endpoints.py | 11 ++ app/flatpages/help/api.md | 66 ++++---- app/tests/unit/test_minetest_hypertext.py | 95 ++++++++++++ app/utils/minetest_hypertext.py | 177 ++++++++++++++++++++++ 4 files changed, 322 insertions(+), 27 deletions(-) create mode 100644 app/tests/unit/test_minetest_hypertext.py create mode 100644 app/utils/minetest_hypertext.py diff --git a/app/blueprints/api/endpoints.py b/app/blueprints/api/endpoints.py index ed46d72c..00583e9d 100644 --- a/app/blueprints/api/endpoints.py +++ b/app/blueprints/api/endpoints.py @@ -36,6 +36,7 @@ from . import bp from .auth import is_api_authd from .support import error, api_create_vcs_release, api_create_zip_release, api_create_screenshot, \ api_order_screenshots, api_edit_package, api_set_cover_image +from ...utils.minetest_hypertext import html_to_minetest def cors_allowed(f): @@ -84,6 +85,16 @@ def package(package): return jsonify(package.getAsDictionary(current_app.config["BASE_URL"])) +@bp.route("/api/packages///hypertext/") +@is_package_page +@cors_allowed +def package_hypertext(package): + formspec_version = request.args["formspec_version"] + include_images = isYes(request.args.get("include_images", "true")) + html = render_markdown(package.desc) + return jsonify(html_to_minetest(html, formspec_version, include_images)) + + @bp.route("/api/packages///", methods=["PUT"]) @csrf.exempt @is_package_page diff --git a/app/flatpages/help/api.md b/app/flatpages/help/api.md index 238b2db4..6cfd9697 100644 --- a/app/flatpages/help/api.md +++ b/app/flatpages/help/api.md @@ -8,7 +8,7 @@ title: API ## Responses and Error Handling -If there is an error, the response will be JSON similar to the following with a non-200 status code: +If there is an error, the response will be JSON similar to the following with a non-200 status code: ```json { @@ -26,7 +26,7 @@ often other keys with information. For example: { "success": true, "release": { - /* same as returned by a GET */ + /* same as returned by a GET */ } } ``` @@ -39,7 +39,7 @@ the number of items is specified using `num` The response will be a dictionary with the following keys: -* `page`: page number, integer from 1 to max +* `page`: page number, integer from 1 to max * `per_page`: number of items per page, same as `n` * `page_count`: number of pages * `total`: total number of results @@ -55,7 +55,7 @@ Not all endpoints require authentication, but it is done using Bearer tokens: ```bash curl https://content.minetest.net/api/whoami/ \ - -H "Authorization: Bearer YOURTOKEN" + -H "Authorization: Bearer YOURTOKEN" ``` Tokens can be attained by visiting [Settings > API Tokens](/user/tokens/). @@ -83,7 +83,7 @@ Tokens can be attained by visiting [Settings > API Tokens](/user/tokens/). * `tags`: List of [tag](#tags) names. * `content_warnings`: List of [content warning](#content-warnings) names. * `license`: A [license](#licenses) name. - * `media_license`: A [license](#licenses) name. + * `media_license`: A [license](#licenses) name. * `long_description`: Long markdown description. * `repo`: Git repo URL. * `website`: Website URL. @@ -92,15 +92,27 @@ Tokens can be attained by visiting [Settings > API Tokens](/user/tokens/). * `video_url`: URL to a video. * `donate_url`: URL to a donation page. * `game_support`: Array of game support information objects. Not currently documented, as subject to change. +* GET `/api/packages///hypertext/` + * Converts the long description to [Minetest Markup Language](https://github.com/minetest/minetest/blob/master/doc/lua_api.md#markup-language) + to be used in a `hypertext` formspec element. + * Query arguments: + * `formspec_version`: Required, maximum supported formspec version. + * `include_images`: Optional, defaults to true. + * Returns JSON dictionary with following key: + * `head`: markup for suggested styling and custom tags, prepend to the body before displaying. + * `body`: markup for long description. + * `links`: dictionary of anchor name to link URL. + * `images`: dictionary of img name to image URL + * `image_tooltips`: dictionary of img name to tooltip text. * GET `/api/packages///dependencies/` - * Returns dependencies, with suggested candidates + * Returns dependencies, with suggested candidates * If query argument `only_hard` is present, only hard deps will be returned. * GET `/api/dependencies/` * Returns `provides` and raw dependencies for all packages. * Supports [Package Queries](#package-queries) * [Paginated result](#paginated-results), max 300 results per page * Each item in `items` will be a dictionary with the following keys: - * `type`: One of `GAME`, `MOD`, `TXP`. + * `type`: One of `GAME`, `MOD`, `TXP`. * `author`: Username of the package author. * `name`: Package name. * `provides`: List of technical mod names inside the package. @@ -142,7 +154,7 @@ Examples: curl -X PUT https://content.minetest.net/api/packages/username/name/ \ -H "Authorization: Bearer YOURTOKEN" -H "Content-Type: application/json" \ -d '{ "title": "Foo bar", "tags": ["pvp", "survival"], "license": "MIT" }' - + # Remove website URL curl -X PUT https://content.minetest.net/api/packages/username/name/ \ -H "Authorization: Bearer YOURTOKEN" -H "Content-Type: application/json" \ @@ -161,7 +173,7 @@ Supported query parameters: * `q`: Query string. * `author`: Filter by author. * `tag`: Filter by tags. -* `game`: Filter by [Game Support](/help/game_support/), ex: `Wuzzy/mineclone2`. (experimental, doesn't show items that support every game currently). +* `game`: Filter by [Game Support](/help/game_support/), ex: `Wuzzy/mineclone2`. (experimental, doesn't show items that support every game currently). * `random`: When present, enable random ordering and ignore `sort`. * `limit`: Return at most `limit` packages. * `hide`: Hide content based on [Content Flags](/help/content_flags/). @@ -171,12 +183,12 @@ Supported query parameters: * `engine_version`: Only show packages supported by this Minetest engine version, eg: `5.3.0`. * `fmt`: How the response is formatted. * `keys`: author/name only. - * `short`: stuff needed for the Minetest client. + * `short`: stuff needed for the Minetest client. ### Releases -* GET `/api/releases/` (List) +* GET `/api/releases/` (List) * Limited to 30 most recent releases. * Optional arguments: * `author`: Filter by author @@ -204,14 +216,14 @@ Supported query parameters: * For Git release creation: * `method`: must be `git`. * `ref`: (Optional) git reference, eg: `master`. - * For zip upload release creation: + * For zip upload release creation: * `file`: multipart file to upload, like ``. * `commit`: (Optional) Source Git commit hash, for informational purposes. * You can set min and max Minetest Versions [using the content's .conf file](/help/package_config/). * DELETE `/api/packages///releases//` (Delete) * Requires authentication. * Deletes release. - + Examples: ```bash @@ -232,7 +244,7 @@ curl -X POST https://content.minetest.net/api/packages/username/name/releases/ne # Delete release curl -X DELETE https://content.minetest.net/api/packages/username/name/releases/3/ \ - -H "Authorization: Bearer YOURTOKEN" + -H "Authorization: Bearer YOURTOKEN" ``` @@ -275,7 +287,7 @@ Examples: curl -X POST https://content.minetest.net/api/packages/username/name/screenshots/new/ \ -H "Authorization: Bearer YOURTOKEN" \ -F title="My Release" -F file=@path/to/screnshot.png - + # Create screenshot and set it as the cover image curl -X POST https://content.minetest.net/api/packages/username/name/screenshots/new/ \ -H "Authorization: Bearer YOURTOKEN" \ @@ -283,13 +295,13 @@ curl -X POST https://content.minetest.net/api/packages/username/name/screenshots # Delete screenshot curl -X DELETE https://content.minetest.net/api/packages/username/name/screenshots/3/ \ - -H "Authorization: Bearer YOURTOKEN" - + -H "Authorization: Bearer YOURTOKEN" + # Reorder screenshots curl -X POST https://content.minetest.net/api/packages/username/name/screenshots/order/ \ -H "Authorization: Bearer YOURTOKEN" -H "Content-Type: application/json" \ -d "[13, 2, 5, 7]" - + # Set cover image curl -X POST https://content.minetest.net/api/packages/username/name/screenshots/cover-image/ \ -H "Authorization: Bearer YOURTOKEN" -H "Content-Type: application/json" \ @@ -302,7 +314,7 @@ curl -X POST https://content.minetest.net/api/packages/username/name/screenshots * GET `/api/packages///reviews/` (List) * Returns array of review dictionaries with keys: * `user`: dictionary with `display_name` and `username`. - * `title`: review title + * `title`: review title * `comment`: the text * `rating`: 1 for negative, 3 for neutral, 5 for positive * `is_positive`: boolean @@ -326,16 +338,16 @@ Example: ```json [ { - "comment": "This is a really good mod!", - "created_at": "2021-11-24T16:18:33.764084", - "is_positive": true, - "title": "Really good", + "comment": "This is a really good mod!", + "created_at": "2021-11-24T16:18:33.764084", + "is_positive": true, + "title": "Really good", "user": { - "display_name": "rubenwardy", + "display_name": "rubenwardy", "username": "rubenwardy" - }, + }, "votes": { - "helpful": 0, + "helpful": 0, "unhelpful": 0 } } @@ -403,7 +415,7 @@ Supported query parameters: * `description`: tag description or null. * `is_protected`: boolean, whether the tag is protected (can only be set by Editors in the web interface). * `views`: number of views of this tag. - + ### Content Warnings * GET `/api/content_warnings/` ([View](/api/content_warnings/)): List of: diff --git a/app/tests/unit/test_minetest_hypertext.py b/app/tests/unit/test_minetest_hypertext.py new file mode 100644 index 00000000..ce2b832a --- /dev/null +++ b/app/tests/unit/test_minetest_hypertext.py @@ -0,0 +1,95 @@ +from app.utils.minetest_hypertext import html_to_minetest + + +conquer_html = """ +

+Welcome to Conquer, a mod that adds RTS gameplay. It allows players to start +Conquer sub-games, where they can place buildings, train units, and +fight other players. +

+ +

Starting or joining a session

+ +

+You can list running sessions by typing: +

+ +
/conquer list
+new line
+
+another new line
+ +

+You'll switch into Conquer playing mode, where you will be given buildings that you can place. +You'll need to place a keep, which you must protect at all costs. +

+ +

+You may leave a game and return to normal playing mode at anytime by typing: +

+ +

Conquer GUI

+ +

+The Conquer GUI is the central place for monitoring your kingdom. +Once in a session, you can view it by pressing the inventory key (I), +or by punching/right-clicking the keep node. +

+""" + + +conquer_expected = """ +Welcome to Conquer, a mod that adds RTS gameplay. It allows players to start Conquer sub-games, where they can place buildings, train units, and fight other players. + +Starting or joining a session +You can list running sessions by typing: +/conquer list +new line + +another new line +You'll switch into Conquer playing mode, where you will be given buildings that you can place. You'll need to place a keep, which you must protect at all costs. +You may leave a game and return to normal playing mode at anytime by typing: + +Conquer GUI +The Conquer GUI is the central place for monitoring your kingdom. Once in a session, you can view it by pressing the inventory key (I), or by punching/right-clicking the keep node. +""" + + +def test_conquer(): + assert html_to_minetest(conquer_html)["body"].strip() == conquer_expected.strip() + + +def test_images(): + html = """ + + """ + + expected = "" + result = html_to_minetest(html) + assert result["body"].strip() == expected.strip() + assert len(result["images"]) == 1 + assert result["images"]["image_0"] == "/path/to/img.png" + + +def test_bullets(): + html = """ +
    +
  • One
  • +
  • two three
  • +
  • four
  • +
+ """ + + expected = "• One\n• two three\n• four\n" + result = html_to_minetest(html) + assert result["body"].strip() == expected.strip() + + +def test_inline(): + html = """ + One two three + """ + + expected = "One two three" + result = html_to_minetest(html) + assert result["body"].strip() == expected.strip() diff --git a/app/utils/minetest_hypertext.py b/app/utils/minetest_hypertext.py new file mode 100644 index 00000000..25068cdf --- /dev/null +++ b/app/utils/minetest_hypertext.py @@ -0,0 +1,177 @@ +from html.parser import HTMLParser +import re +import sys + + +def normalize_whitespace(x): + return re.sub(r"\s+", " ", x) + + +assert normalize_whitespace(" one three\nfour\n\n") == " one three four " + + +# Styles and custom tags +HEAD = normalize_whitespace(""" + + +""").strip() + + +def get_attributes(attrs): + retval = {} + for attr in attrs: + retval[attr[0]] = attr[1] + return retval + + +class MinetestHTMLParser(HTMLParser): + def __init__(self, include_images): + super().__init__() + self.include_images = include_images + + self.text_buffer = "" + self.has_line_started = False + self.links = {} + self.images = {} + self.image_tooltips = {} + self.is_preserving = False + self.remove_until = None + + def handle_starttag(self, tag, attrs): + if self.is_preserving or self.remove_until: + return + + print("OPEN", tag, file=sys.stderr) + + self.has_line_started = True + if tag == "p": + self.has_line_started = False + elif tag == "pre": + self.text_buffer += "" + self.is_preserving = True + self.has_line_started = False + elif tag == "table": + # Tables are currently unsupported and removed + self.remove_until = "table" + self.text_buffer += "(table removed)\n" + elif tag == "br": + self.text_buffer += "\n" + self.has_line_started = False + elif tag == "h1" or tag == "h2": + self.text_buffer += "\n" + elif tag == "h3" or tag == "h4" or tag == "h5": + self.text_buffer += "\n" + elif tag == "a": + for attr in attrs: + if attr[0] == "href": + name = f"link_{len(self.links)}" + self.links[name] = attr[1] + self.text_buffer += f"" + break + else: + self.text_buffer += "" + elif tag == "img": + attr_by_value = get_attributes(attrs) + if "src" in attr_by_value and self.include_images: + name = f"image_{len(self.images)}" + self.images[name] = attr_by_value["src"] + width = attr_by_value.get("width", 128) + height = attr_by_value.get("height", 128) + self.text_buffer += f"" + + if "alt" in attr_by_value: + self.image_tooltips[name] = attr_by_value["alt"] + elif tag == "b" or tag == "strong": + self.text_buffer += "" + elif tag == "i" or tag == "em": + self.text_buffer += "" + elif tag == "u": + self.text_buffer += "" + elif tag == "li": + self.has_line_started = False + self.text_buffer += "• " + elif tag == "code": + self.text_buffer += "" + elif tag == "span" or tag == "ul": + pass + else: + print("UNKNOWN TAG ", tag, attrs, file=sys.stderr) + + def handle_endtag(self, tag): + if self.remove_until: + if self.remove_until == tag: + self.remove_until = None + return + + print("CLOSE", tag, file=sys.stderr) + + if tag == "pre": + self.text_buffer = self.text_buffer.rstrip() + self.text_buffer += "\n" + self.is_preserving = False + self.has_line_started = False + elif self.is_preserving: + return + elif tag == "p": + self.text_buffer = self.text_buffer.rstrip() + self.text_buffer += "\n" + self.has_line_started = False + elif tag == "h1" or tag == "h2": + self.text_buffer += "\n" + self.has_line_started = False + elif tag == "h3" or tag == "h4" or tag == "h5": + self.text_buffer += "\n" + self.has_line_started = False + elif tag == "a": + self.text_buffer += "" + elif tag == "code": + self.text_buffer += "" + elif tag == "b" or tag == "strong": + self.text_buffer += "" + elif tag == "i" or tag == "em": + self.text_buffer += "" + elif tag == "u": + self.text_buffer += "" + elif tag == "li": + self.text_buffer += "\n" + # else: + # print("END", tag, file=sys.stderr) + + def handle_data(self, data): + print(f"DATA \"{data}\"", file=sys.stderr) + if self.remove_until: + return + + if not self.is_preserving: + data = normalize_whitespace(data) + if not self.has_line_started: + data = data.lstrip() + + self.text_buffer += data + self.has_line_started = True + + def handle_entityref(self, name): + to_value = { + "lt": "\\<", + "gr": "\\>", + "amp": "&", + "quot": "\"", + "apos": "'", + } + + if name in to_value: + self.text_buffer += to_value[name] + else: + self.text_buffer += f"&{name};" + + +def html_to_minetest(html, formspec_version=6, include_images=True): + parser = MinetestHTMLParser(include_images) + parser.feed(html) + return { + "head": HEAD, + "body": parser.text_buffer.strip() + "\n\n", + "links": parser.links, + "images": parser.images, + "image_tooltips": parser.image_tooltips, + }