Import forum profile pictures and host them directly

app/blueprints/admin/actions.py

@@ -27,9 +27,9 @@ from app.logic.game_support import GameSupportResolver
 from app.models import PackageRelease, db, Package, PackageState, PackageScreenshot, MetaPackage, User, \
 	NotificationType, PackageUpdateConfig, License, UserRank, PackageType, ThreadReply
 from app.tasks.emails import send_pending_digests
-from app.tasks.forumtasks import importTopicList, checkAllForumAccounts
+from app.tasks.forumtasks import importTopicList, checkAllForumAccounts, checkForumAccount
 from app.tasks.importtasks import importRepoScreenshot, checkZipRelease, check_for_updates, updateAllGameSupport
-from app.tasks.usertasks import upgrade_new_members
+from app.tasks.usertasks import upgrade_new_members, set_profile_picture_from_url
 from app.utils import addNotification, get_system_user
 from app.utils.image import get_image_size
 
@@ -102,6 +102,13 @@ def check_all_forum_accounts():
 	return redirect(url_for("tasks.check", id=task.id, r=url_for("admin.admin_page")))
 
 
+@action("Import forum profile pics")
+def import_forum_profile_pics():
+	users = User.query.filter(and_(User.forums_username.isnot(None), User.profile_pic.ilike("https://forum.minetest.net/%"))).all()
+	for user in users:
+		checkForumAccount.delay(user.forums_username)
+
+
 @action("Import screenshots from Git")
 def import_screenshots():
 	packages = Package.query \
@@ -129,8 +136,9 @@ def clean_uploads():
 
 		release_urls = get_filenames_from_column(PackageRelease.url)
 		screenshot_urls = get_filenames_from_column(PackageScreenshot.url)
+		pp_urls = get_filenames_from_column(User.profile_pic)
 
-		db_urls = release_urls.union(screenshot_urls)
+		db_urls = release_urls.union(screenshot_urls).union(pp_urls)
 		unreachable = existing_uploads.difference(db_urls)
 
 		import sys
@@ -343,3 +351,12 @@ def set_new_members():
 	task_id = uuid()
 	upgrade_new_members.apply_async((), task_id=task_id)
 	return redirect(url_for("tasks.check", id=task_id, r=url_for("admin.admin_page")))
+
+
+@action("Import profile pictures from forums")
+def import_forum_pp():
+	users = User.query.filter(User.profile_pic.ilike("https://forum.minetest.net/%")).all()
+	for user in users:
+		set_profile_picture_from_url.delay(user.username, user.profile_pic)
+
+	flash(f"Importing {len(users)} profile pictures", "success")

app/blueprints/users/profile.py

@@ -235,25 +235,6 @@ def profile(username):
 			medals_unlocked=unlocked, medals_locked=locked)
 
 
-@bp.route("/users/<username>/check/", methods=["POST"])
-@login_required
-def user_check(username):
-	user = User.query.filter_by(username=username).first()
-	if user is None:
-		abort(404)
-
-	if current_user != user and not current_user.rank.atLeast(UserRank.MODERATOR):
-		abort(403)
-
-	if user.forums_username is None:
-		abort(404)
-
-	task = checkForumAccount.delay(user.forums_username)
-	next_url = url_for("users.profile", username=username)
-
-	return redirect(url_for("tasks.check", id=task.id, r=next_url))
-
-
 @bp.route("/user/stats/")
 @login_required
 def statistics_redirect():

app/tasks/forumtasks.py

@@ -21,25 +21,29 @@ from app.tasks import celery
 from app.utils import is_username_valid
 from app.utils.phpbbparser import getProfile, getTopicsFromForum
 import urllib.request
+from urllib.parse import urljoin
+from .usertasks import set_profile_picture_from_url
+
 
 @celery.task()
-def checkForumAccount(username, forceNoSave=False):
-	print("Checking " + username)
+def checkForumAccount(forums_username):
+	print("### Checking " + forums_username, file=sys.stderr)
 	try:
-		profile = getProfile("https://forum.minetest.net", username)
-	except OSError:
+		profile = getProfile("https://forum.minetest.net", forums_username)
+	except OSError as e:
+		print(e, file=sys.stderr)
 		return
 
 	if profile is None:
 		return
 
-	user = User.query.filter_by(forums_username=username).first()
+	user = User.query.filter_by(forums_username=forums_username).first()
 
 	# Create user
 	needsSaving = False
 	if user is None:
-		user = User(username)
-		user.forums_username = username
+		user = User(forums_username)
+		user.forums_username = forums_username
 		db.session.add(user)
 
 	# Get github username
@@ -50,33 +54,32 @@ def checkForumAccount(username, forceNoSave=False):
 		needsSaving = True
 
 	pic = profile.avatar
-	if pic and "http" in pic:
+	if pic and pic.startswith("http"):
 		pic = None
 
-	needsSaving = needsSaving or pic != user.profile_pic
-	if pic:
-		user.profile_pic = "https://forum.minetest.net/" + pic
-	else:
-		user.profile_pic = None
-
 	# Save
-	if needsSaving and not forceNoSave:
+	if needsSaving:
 		db.session.commit()
 
+	if pic:
+		pic = urljoin("https://forum.minetest.net/", pic)
+		print(f"####### Picture: {pic}", file=sys.stderr)
+		print(f"####### User pp {user.profile_pic}", file=sys.stderr)
+
+		pic_needs_replacing = user.profile_pic is None or user.profile_pic == "" or \
+				user.profile_pic.startswith("https://forum.minetest.net")
+		if pic_needs_replacing and pic.startswith("https://forum.minetest.net"):
+			print(f"####### Queueing", file=sys.stderr)
+			set_profile_picture_from_url.delay(user.username, pic)
+
 	return needsSaving
 
 
 @celery.task()
-def checkAllForumAccounts(forceNoSave=False):
-	needsSaving = False
+def checkAllForumAccounts():
 	query = User.query.filter(User.forums_username.isnot(None))
 	for user in query.all():
-		needsSaving = checkForumAccount(user.username) or needsSaving
-
-	if needsSaving and not forceNoSave:
-		db.session.commit()
-
-	return needsSaving
+		checkForumAccount(user.forums_username)
 
 
 regex_tag    = re.compile(r"\[([a-z0-9_]+)\]")

app/tasks/usertasks.py

@@ -1,5 +1,5 @@
 # ContentDB
-# Copyright (C) 2021 rubenwardy
+# Copyright (C) 2021-23 rubenwardy
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -15,13 +15,17 @@
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
 
-import datetime
+import datetime, requests
+import os
+import sys
 
 from sqlalchemy import or_, and_
 
+from app import app
 from app.models import User, db, UserRank, ThreadReply, Package
+from app.utils import randomString
 from app.utils.models import create_session
-from app.tasks import celery
+from app.tasks import celery, TaskError
 
 
 @celery.task()
@@ -46,3 +50,45 @@ def upgrade_new_members():
 				User.packages.any(Package.approved_at < threshold)))).update({"rank": UserRank.MEMBER}, synchronize_session=False)
 
 		session.commit()
+
+
+@celery.task()
+def set_profile_picture_from_url(username: str, url: str):
+	print("### Setting pp for " + username + " to " + url, file=sys.stderr)
+	user = User.query.filter_by(username=username).first()
+	if user is None:
+		raise TaskError(f"Unable to find user {username}")
+
+	headers = {"Accept": "image/jpeg, image/png, image/gif"}
+	resp = requests.get(url, stream=True, headers=headers, timeout=15)
+	if resp.status_code != 200:
+		raise TaskError(f"Failed to download {url}: {resp.status_code}: {resp.reason}")
+
+	content_type = resp.headers["content-type"]
+	if content_type is None:
+		raise TaskError("Content-Type needed")
+	elif content_type == "image/jpeg":
+		ext = "jpg"
+	elif content_type == "image/png":
+		ext = "png"
+	elif content_type == "image/gif":
+		ext = "gif"
+	else:
+		raise TaskError(f"Unacceptable content-type: {content_type}")
+
+	filename = randomString(10) + "." + ext
+	filepath = os.path.join(app.config["UPLOAD_DIR"], filename)
+	with open(filepath, "wb") as f:
+		size = 0
+		for chunk in resp.iter_content(chunk_size=1024):
+			if chunk:  # filter out keep-alive new chunks
+				size += len(chunk)
+				if size > 3 * 1000 * 1000:  # 3 MB
+					raise TaskError(f"File too large to download {url}")
+
+				f.write(chunk)
+
+	user.profile_pic = "/uploads/" + filename
+	db.session.commit()
+
+	return filepath

app/templates/users/profile_edit.html

@@ -21,12 +21,6 @@
 		{% endif %}
 	</div>
 	<div class="col">
-		{% if user.forums_username %}
-			<form method="post" action="{{ url_for('users.user_check', username=user.username) }}" class="" style="display:inline-block;">
-				<input type="hidden" name="csrf_token" value="{{ csrf_token() }}" />
-				<input type="submit" class="btn btn-primary" value="{{  _('Sync with Forums') }}" />
-			</form>
-		{% endif %}
 		{% if user.email %}
 			<a class="btn btn-primary" href="https://en.gravatar.com/">
 				Gravatar

app/utils/phpbbparser.py

@@ -92,7 +92,7 @@ def getProfile(url, username):
 	url = getProfileURL(url, username)
 
 	try:
-		req = urllib.request.urlopen(url, timeout=5)
+		req = urllib.request.urlopen(url, timeout=15)
 	except urllib.error.HTTPError as e:
 		if e.code == 404:
 			return None