From 658d319eb04d8a970befd325f72af3a769b706da Mon Sep 17 00:00:00 2001 From: rubenwardy Date: Sun, 25 Feb 2024 16:59:51 +0000 Subject: [PATCH] Add translation importing to post_release_check_update --- app/tasks/importtasks.py | 34 ++++++++++++++++++-- app/tasks/minetestcheck/tree.py | 55 ++++++++++++++++++++++----------- 2 files changed, 68 insertions(+), 21 deletions(-) diff --git a/app/tasks/importtasks.py b/app/tasks/importtasks.py index 5f917227..2b4608e5 100644 --- a/app/tasks/importtasks.py +++ b/app/tasks/importtasks.py @@ -27,15 +27,16 @@ from git import GitCommandError from git_archive_all import GitArchiver from kombu import uuid from sqlalchemy import and_ +from sqlalchemy.dialects.postgresql import insert from app.models import AuditSeverity, db, NotificationType, PackageRelease, MetaPackage, Dependency, PackageType, \ MinetestRelease, Package, PackageState, PackageScreenshot, PackageUpdateTrigger, PackageUpdateConfig, \ - PackageGameSupport + PackageGameSupport, PackageTranslation, Language from app.tasks import celery, TaskError from app.utils import random_string, post_bot_message, add_system_notification, add_system_audit_log, \ get_games_from_list, add_audit_log from app.utils.git import clone_repo, get_latest_tag, get_latest_commit, get_temp_dir -from .minetestcheck import build_tree, MinetestCheckError, ContentType +from .minetestcheck import build_tree, MinetestCheckError, ContentType, PackageTreeNode from .webhooktasks import post_discord_webhook from app import app from app.logic.LogicError import LogicError @@ -96,7 +97,7 @@ def update_all_game_support(): def post_release_check_update(self, release: PackageRelease, path): try: - tree = build_tree(path, expected_type=ContentType[release.package.type.name], + tree: PackageTreeNode = build_tree(path, expected_type=ContentType[release.package.type.name], author=release.package.author.username, name=release.package.name) if tree.name is not None and release.package.name != tree.name and tree.type == ContentType.MOD: @@ -162,6 +163,33 @@ def post_release_check_update(self, release: PackageRelease, path): for meta in get_meta_packages(optional_depends): db.session.add(Dependency(package, meta=meta, optional=True)) + # Read translations + allowed_languages = set([x[0] for x in db.session.query(Language.id).all()]) + allowed_languages.discard("en") + raw_translations = tree.get_translations(tree.get("textdomain", tree.name)) + conn = db.session.connection() + for raw_translation in raw_translations: + if raw_translation.language not in allowed_languages: + continue + + to_update = { + "title": raw_translation.entries.get(tree.get("title", package.title)), + "short_desc": raw_translation.entries.get(tree.get("description", package.short_desc)), + } + values = { + "package_id": package.id, + "language_id": raw_translation.language, + "title": to_update["title"], + "short_desc": to_update["short_desc"], + } + + stmt = insert(PackageTranslation).values(**values) + stmt = stmt.on_conflict_do_update( + index_elements=[PackageTranslation.package_id, PackageTranslation.language_id], + set_=to_update + ) + conn.execute(stmt) + # Update min/max if tree.meta.get("min_minetest_version"): release.min_rel = MinetestRelease.get(tree.meta["min_minetest_version"], None) diff --git a/app/tasks/minetestcheck/tree.py b/app/tasks/minetestcheck/tree.py index 4f3790dd..144a923c 100644 --- a/app/tasks/minetestcheck/tree.py +++ b/app/tasks/minetestcheck/tree.py @@ -17,10 +17,12 @@ import os import re +import glob from typing import Optional from . import MinetestCheckError, ContentType from .config import parse_conf +from .translation import Translation, parse_tr basenamePattern = re.compile("^([a-z0-9_]+)$") licensePattern = re.compile("^(licen[sc]e|copying)(.[^/\n]+)?$", re.IGNORECASE) @@ -31,7 +33,7 @@ DISALLOWED_NAMES = { } -def get_base_dir(path): +def get_base_dir(path) -> str: if not os.path.isdir(path): raise IOError("Expected dir") @@ -42,7 +44,7 @@ def get_base_dir(path): return path -def detect_type(path): +def detect_type(path) -> ContentType: if os.path.isfile(path + "/game.conf"): return ContentType.GAME elif os.path.isfile(path + "/init.lua"): @@ -58,7 +60,7 @@ def detect_type(path): return ContentType.UNKNOWN -def get_csv_line(line): +def get_csv_line(line) -> list[str]: if line is None: return [] @@ -79,23 +81,33 @@ def check_name_list(key: str, value: list[str], relative: str, allow_star: bool class PackageTreeNode: - def __init__(self, base_dir, relative, author=None, repo=None, name=None): - self.baseDir = base_dir + baseDir: str + relative: str + author: Optional[str] + name: Optional[str] + repo: Optional[str] + meta: dict + children: list + type: ContentType + + def __init__(self, base_dir: str, relative: str, + author: Optional[str] = None, repo: Optional[str] = None, name: Optional[str] = None): + self.baseDir = base_dir self.relative = relative - self.author = author - self.name = name - self.repo = repo - self.meta = None + self.author = author + self.name = name + self.repo = repo + self.meta = {} self.children = [] # Detect type self.type = detect_type(base_dir) - self.read_meta() + self._read_meta() if self.type == ContentType.GAME: if not os.path.isdir(os.path.join(base_dir, "mods")): raise MinetestCheckError("Game at {} does not have a mods/ folder".format(self.relative)) - self.add_children_from_mod_dir("mods") + self._add_children_from_mod_dir("mods") elif self.type == ContentType.MOD: if self.name and not basenamePattern.match(self.name): raise MinetestCheckError(f"Invalid base name for mod {self.name} at {self.relative}, names must only contain a-z0-9_.") @@ -103,9 +115,9 @@ class PackageTreeNode: if self.name and self.name in DISALLOWED_NAMES: raise MinetestCheckError(f"Forbidden mod name '{self.name}' used at {self.relative}") - self.check_dir_casing(["textures", "media", "sounds", "models", "locale"]) + self._check_dir_casing(["textures", "media", "sounds", "models", "locale"]) elif self.type == ContentType.MODPACK: - self.add_children_from_mod_dir(None) + self._add_children_from_mod_dir(None) def find_license_file(self) -> Optional[str]: for name in os.listdir(self.baseDir): @@ -115,7 +127,7 @@ class PackageTreeNode: return None - def check_dir_casing(self, dirs): + def _check_dir_casing(self, dirs): for dir in next(os.walk(self.baseDir))[1]: lowercase = dir.lower() if lowercase != dir and lowercase in dirs: @@ -139,7 +151,7 @@ class PackageTreeNode: else: return None - def read_meta(self): + def _read_meta(self): result = {} # Read .conf file @@ -229,7 +241,7 @@ class PackageTreeNode: self.meta = result - def add_children_from_mod_dir(self, subdir): + def _add_children_from_mod_dir(self, subdir): dir = self.baseDir relative = self.relative if subdir: @@ -282,9 +294,16 @@ class PackageTreeNode: return retval - def get(self, key): - return self.meta.get(key) + def get(self, key: str, default=None): + return self.meta.get(key, default) def validate(self): for child in self.children: child.validate() + + def get_translations(self, textdomain: str) -> list[Translation]: + ret = [] + + for name in glob.glob(f"{self.baseDir}/**/locale/{textdomain}.*.tr", recursive=True): + ret.append(parse_tr(name)) + return ret