From 3ccb1655223f53e5bcf1deb0419ca7a8264772b2 Mon Sep 17 00:00:00 2001 From: rubenwardy Date: Fri, 5 May 2023 17:14:23 +0100 Subject: [PATCH] Import forum profile pictures and host them directly --- app/blueprints/admin/actions.py | 23 ++++++++++-- app/blueprints/users/profile.py | 19 ---------- app/tasks/forumtasks.py | 49 +++++++++++++------------ app/tasks/usertasks.py | 52 +++++++++++++++++++++++++-- app/templates/users/profile_edit.html | 6 ---- app/utils/phpbbparser.py | 2 +- 6 files changed, 96 insertions(+), 55 deletions(-) diff --git a/app/blueprints/admin/actions.py b/app/blueprints/admin/actions.py index 466053d6..396f8e60 100644 --- a/app/blueprints/admin/actions.py +++ b/app/blueprints/admin/actions.py @@ -27,9 +27,9 @@ from app.logic.game_support import GameSupportResolver from app.models import PackageRelease, db, Package, PackageState, PackageScreenshot, MetaPackage, User, \ NotificationType, PackageUpdateConfig, License, UserRank, PackageType, ThreadReply from app.tasks.emails import send_pending_digests -from app.tasks.forumtasks import importTopicList, checkAllForumAccounts +from app.tasks.forumtasks import importTopicList, checkAllForumAccounts, checkForumAccount from app.tasks.importtasks import importRepoScreenshot, checkZipRelease, check_for_updates, updateAllGameSupport -from app.tasks.usertasks import upgrade_new_members +from app.tasks.usertasks import upgrade_new_members, set_profile_picture_from_url from app.utils import addNotification, get_system_user from app.utils.image import get_image_size @@ -102,6 +102,13 @@ def check_all_forum_accounts(): return redirect(url_for("tasks.check", id=task.id, r=url_for("admin.admin_page"))) +@action("Import forum profile pics") +def import_forum_profile_pics(): + users = User.query.filter(and_(User.forums_username.isnot(None), User.profile_pic.ilike("https://forum.minetest.net/%"))).all() + for user in users: + checkForumAccount.delay(user.forums_username) + + @action("Import screenshots from Git") def import_screenshots(): packages = Package.query \ @@ -129,8 +136,9 @@ def clean_uploads(): release_urls = get_filenames_from_column(PackageRelease.url) screenshot_urls = get_filenames_from_column(PackageScreenshot.url) + pp_urls = get_filenames_from_column(User.profile_pic) - db_urls = release_urls.union(screenshot_urls) + db_urls = release_urls.union(screenshot_urls).union(pp_urls) unreachable = existing_uploads.difference(db_urls) import sys @@ -343,3 +351,12 @@ def set_new_members(): task_id = uuid() upgrade_new_members.apply_async((), task_id=task_id) return redirect(url_for("tasks.check", id=task_id, r=url_for("admin.admin_page"))) + + +@action("Import profile pictures from forums") +def import_forum_pp(): + users = User.query.filter(User.profile_pic.ilike("https://forum.minetest.net/%")).all() + for user in users: + set_profile_picture_from_url.delay(user.username, user.profile_pic) + + flash(f"Importing {len(users)} profile pictures", "success") diff --git a/app/blueprints/users/profile.py b/app/blueprints/users/profile.py index 069f5e8f..2ba42cd9 100644 --- a/app/blueprints/users/profile.py +++ b/app/blueprints/users/profile.py @@ -235,25 +235,6 @@ def profile(username): medals_unlocked=unlocked, medals_locked=locked) -@bp.route("/users//check/", methods=["POST"]) -@login_required -def user_check(username): - user = User.query.filter_by(username=username).first() - if user is None: - abort(404) - - if current_user != user and not current_user.rank.atLeast(UserRank.MODERATOR): - abort(403) - - if user.forums_username is None: - abort(404) - - task = checkForumAccount.delay(user.forums_username) - next_url = url_for("users.profile", username=username) - - return redirect(url_for("tasks.check", id=task.id, r=next_url)) - - @bp.route("/user/stats/") @login_required def statistics_redirect(): diff --git a/app/tasks/forumtasks.py b/app/tasks/forumtasks.py index 4edcc811..1f75d7ac 100644 --- a/app/tasks/forumtasks.py +++ b/app/tasks/forumtasks.py @@ -21,25 +21,29 @@ from app.tasks import celery from app.utils import is_username_valid from app.utils.phpbbparser import getProfile, getTopicsFromForum import urllib.request +from urllib.parse import urljoin +from .usertasks import set_profile_picture_from_url + @celery.task() -def checkForumAccount(username, forceNoSave=False): - print("Checking " + username) +def checkForumAccount(forums_username): + print("### Checking " + forums_username, file=sys.stderr) try: - profile = getProfile("https://forum.minetest.net", username) - except OSError: + profile = getProfile("https://forum.minetest.net", forums_username) + except OSError as e: + print(e, file=sys.stderr) return if profile is None: return - user = User.query.filter_by(forums_username=username).first() + user = User.query.filter_by(forums_username=forums_username).first() # Create user needsSaving = False if user is None: - user = User(username) - user.forums_username = username + user = User(forums_username) + user.forums_username = forums_username db.session.add(user) # Get github username @@ -50,33 +54,32 @@ def checkForumAccount(username, forceNoSave=False): needsSaving = True pic = profile.avatar - if pic and "http" in pic: + if pic and pic.startswith("http"): pic = None - needsSaving = needsSaving or pic != user.profile_pic - if pic: - user.profile_pic = "https://forum.minetest.net/" + pic - else: - user.profile_pic = None - # Save - if needsSaving and not forceNoSave: + if needsSaving: db.session.commit() + if pic: + pic = urljoin("https://forum.minetest.net/", pic) + print(f"####### Picture: {pic}", file=sys.stderr) + print(f"####### User pp {user.profile_pic}", file=sys.stderr) + + pic_needs_replacing = user.profile_pic is None or user.profile_pic == "" or \ + user.profile_pic.startswith("https://forum.minetest.net") + if pic_needs_replacing and pic.startswith("https://forum.minetest.net"): + print(f"####### Queueing", file=sys.stderr) + set_profile_picture_from_url.delay(user.username, pic) + return needsSaving @celery.task() -def checkAllForumAccounts(forceNoSave=False): - needsSaving = False +def checkAllForumAccounts(): query = User.query.filter(User.forums_username.isnot(None)) for user in query.all(): - needsSaving = checkForumAccount(user.username) or needsSaving - - if needsSaving and not forceNoSave: - db.session.commit() - - return needsSaving + checkForumAccount(user.forums_username) regex_tag = re.compile(r"\[([a-z0-9_]+)\]") diff --git a/app/tasks/usertasks.py b/app/tasks/usertasks.py index 05546b22..bba0b5b4 100644 --- a/app/tasks/usertasks.py +++ b/app/tasks/usertasks.py @@ -1,5 +1,5 @@ # ContentDB -# Copyright (C) 2021 rubenwardy +# Copyright (C) 2021-23 rubenwardy # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License as published by @@ -15,13 +15,17 @@ # along with this program. If not, see . -import datetime +import datetime, requests +import os +import sys from sqlalchemy import or_, and_ +from app import app from app.models import User, db, UserRank, ThreadReply, Package +from app.utils import randomString from app.utils.models import create_session -from app.tasks import celery +from app.tasks import celery, TaskError @celery.task() @@ -46,3 +50,45 @@ def upgrade_new_members(): User.packages.any(Package.approved_at < threshold)))).update({"rank": UserRank.MEMBER}, synchronize_session=False) session.commit() + + +@celery.task() +def set_profile_picture_from_url(username: str, url: str): + print("### Setting pp for " + username + " to " + url, file=sys.stderr) + user = User.query.filter_by(username=username).first() + if user is None: + raise TaskError(f"Unable to find user {username}") + + headers = {"Accept": "image/jpeg, image/png, image/gif"} + resp = requests.get(url, stream=True, headers=headers, timeout=15) + if resp.status_code != 200: + raise TaskError(f"Failed to download {url}: {resp.status_code}: {resp.reason}") + + content_type = resp.headers["content-type"] + if content_type is None: + raise TaskError("Content-Type needed") + elif content_type == "image/jpeg": + ext = "jpg" + elif content_type == "image/png": + ext = "png" + elif content_type == "image/gif": + ext = "gif" + else: + raise TaskError(f"Unacceptable content-type: {content_type}") + + filename = randomString(10) + "." + ext + filepath = os.path.join(app.config["UPLOAD_DIR"], filename) + with open(filepath, "wb") as f: + size = 0 + for chunk in resp.iter_content(chunk_size=1024): + if chunk: # filter out keep-alive new chunks + size += len(chunk) + if size > 3 * 1000 * 1000: # 3 MB + raise TaskError(f"File too large to download {url}") + + f.write(chunk) + + user.profile_pic = "/uploads/" + filename + db.session.commit() + + return filepath diff --git a/app/templates/users/profile_edit.html b/app/templates/users/profile_edit.html index 67020729..4e031d11 100644 --- a/app/templates/users/profile_edit.html +++ b/app/templates/users/profile_edit.html @@ -21,12 +21,6 @@ {% endif %}
- {% if user.forums_username %} -
- - -
- {% endif %} {% if user.email %} Gravatar diff --git a/app/utils/phpbbparser.py b/app/utils/phpbbparser.py index ab16705e..420dac52 100644 --- a/app/utils/phpbbparser.py +++ b/app/utils/phpbbparser.py @@ -92,7 +92,7 @@ def getProfile(url, username): url = getProfileURL(url, username) try: - req = urllib.request.urlopen(url, timeout=5) + req = urllib.request.urlopen(url, timeout=15) except urllib.error.HTTPError as e: if e.code == 404: return None