Refactor admin page

app/blueprints/admin/actions.py

@@ -0,0 +1,210 @@
+# ContentDB
+# Copyright (C) 2018-21 rubenwardy
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+
+import os
+
+from celery import group
+from flask import *
+from sqlalchemy import or_
+
+from app.models import *
+from app.tasks.forumtasks import importTopicList, checkAllForumAccounts
+from app.tasks.importtasks import importRepoScreenshot, checkZipRelease, check_for_updates
+from app.utils import addNotification, get_system_user
+
+actions = {}
+
+def action(title: str):
+	def func(f):
+		name = f.__name__
+		actions[name] = {
+			"title": title,
+			"func": f,
+		}
+
+		return f
+
+	return func
+
+@action("Delete stuck releases")
+def delstuckreleases():
+	PackageRelease.query.filter(PackageRelease.task_id != None).delete()
+	db.session.commit()
+	return redirect(url_for("admin.admin_page"))
+
+@action("Check releases")
+def check_releases():
+	releases = PackageRelease.query.filter(PackageRelease.url.like("/uploads/%")).all()
+
+	tasks = []
+	for release in releases:
+		zippath = release.url.replace("/uploads/", app.config["UPLOAD_DIR"])
+		tasks.append(checkZipRelease.s(release.id, zippath))
+
+	result = group(tasks).apply_async()
+
+	while not result.ready():
+		import time
+		time.sleep(0.1)
+
+	return redirect(url_for("todo.view_editor"))
+
+@action("Reimport packages")
+def reimport_packages():
+	tasks = []
+	for package in Package.query.filter(Package.state!=PackageState.DELETED).all():
+		release = package.releases.first()
+		if release:
+			zippath = release.url.replace("/uploads/", app.config["UPLOAD_DIR"])
+			tasks.append(checkZipRelease.s(release.id, zippath))
+
+	result = group(tasks).apply_async()
+
+	while not result.ready():
+		import time
+		time.sleep(0.1)
+
+	return redirect(url_for("todo.view_editor"))
+
+@action("Import topic list")
+def import_topic_list():
+	task = importTopicList.delay()
+	return redirect(url_for("tasks.check", id=task.id, r=url_for("todo.topics")))
+
+@action("Check all forum accounts")
+def check_all_forum_accounts():
+	task = checkAllForumAccounts.delay()
+	return redirect(url_for("tasks.check", id=task.id, r=url_for("admin.admin_page")))
+
+@action("Import screenshots")
+def import_screenshots():
+	packages = Package.query \
+		.filter(Package.state!=PackageState.DELETED) \
+		.outerjoin(PackageScreenshot, Package.id==PackageScreenshot.package_id) \
+		.filter(PackageScreenshot.id==None) \
+		.all()
+	for package in packages:
+		importRepoScreenshot.delay(package.id)
+
+	return redirect(url_for("admin.admin_page"))
+
+@action("Clean uploads")
+def cleanuploads():
+	upload_dir = app.config['UPLOAD_DIR']
+
+	(_, _, filenames) = next(os.walk(upload_dir))
+	existing_uploads = set(filenames)
+
+	if len(existing_uploads) != 0:
+		def getURLsFromDB(column):
+			results = db.session.query(column).filter(column != None, column != "").all()
+			return set([os.path.basename(x[0]) for x in results])
+
+		release_urls = getURLsFromDB(PackageRelease.url)
+		screenshot_urls = getURLsFromDB(PackageScreenshot.url)
+
+		db_urls = release_urls.union(screenshot_urls)
+		unreachable = existing_uploads.difference(db_urls)
+
+		import sys
+		print("On Disk: ", existing_uploads, file=sys.stderr)
+		print("In DB: ", db_urls, file=sys.stderr)
+		print("Unreachable: ", unreachable, file=sys.stderr)
+
+		for filename in unreachable:
+			os.remove(os.path.join(upload_dir, filename))
+
+		flash("Deleted " + str(len(unreachable)) + " unreachable uploads", "success")
+	else:
+		flash("No downloads to create", "danger")
+
+	return redirect(url_for("admin.admin_page"))
+
+@action("Delete metapackages")
+def delmetapackages():
+	query = MetaPackage.query.filter(~MetaPackage.dependencies.any(), ~MetaPackage.packages.any())
+	count = query.count()
+	query.delete(synchronize_session=False)
+	db.session.commit()
+
+	flash("Deleted " + str(count) + " unused meta packages", "success")
+	return redirect(url_for("admin.admin_page"))
+
+@action("Delete removed packages")
+def delremovedpackages():
+	query = Package.query.filter_by(state=PackageState.DELETED)
+	count = query.count()
+	for pkg in query.all():
+		pkg.review_thread = None
+		db.session.delete(pkg)
+	db.session.commit()
+
+	flash("Deleted {} soft deleted packages packages".format(count), "success")
+	return redirect(url_for("admin.admin_page"))
+
+@action("Add update config")
+def addupdateconfig():
+	added = 0
+	for pkg in Package.query.filter(Package.repo != None, Package.releases.any(), Package.update_config == None).all():
+		pkg.update_config = PackageUpdateConfig()
+		pkg.update_config.auto_created = True
+
+		release: PackageRelease = pkg.releases.first()
+		if release and release.commit_hash:
+			pkg.update_config.last_commit = release.commit_hash
+
+		db.session.add(pkg.update_config)
+		added += 1
+
+	db.session.commit()
+
+	flash("Added {} update configs".format(added), "success")
+	return redirect(url_for("admin.admin_page"))
+
+@action("Run update configs")
+def runupdateconfig():
+	check_for_updates.delay()
+
+	flash("Started update configs", "success")
+	return redirect(url_for("admin.admin_page"))
+
+@action("Send WIP package notification")
+def remindwip():
+	users = User.query.filter(User.packages.any(or_(Package.state==PackageState.WIP, Package.state==PackageState.CHANGES_NEEDED)))
+	system_user = get_system_user()
+	for user in users:
+		packages = db.session.query(Package.title).filter(
+				Package.author_id==user.id,
+				or_(Package.state==PackageState.WIP, Package.state==PackageState.CHANGES_NEEDED)) \
+			.all()
+
+		# Who needs translations?
+		packages = [pkg[0] for pkg in packages]
+		if len(packages) >= 3:
+			packages[len(packages) - 1] = "and " + packages[len(packages) - 1]
+			packages_list = ", ".join(packages)
+		else:
+			packages_list = "and ".join(packages)
+
+		havent = "haven't" if len(packages) > 1 else "hasn't"
+		if len(packages_list) + 54  > 100:
+			packages_list = packages_list[0:(100-54-1)] + "…"
+
+		addNotification(user, system_user, NotificationType.PACKAGE_APPROVAL,
+			f"Did you forget? {packages_list} {havent} been submitted for review yet",
+				url_for('todo.view_user', username=user.username))
+	db.session.commit()

app/blueprints/admin/admin.py

@@ -14,22 +14,15 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
-
-import os
-
-from celery import group
 from flask import *
 from flask_login import current_user, login_user
 from flask_wtf import FlaskForm
-from sqlalchemy import or_
 from wtforms import *
 from wtforms.validators import InputRequired, Length
-
-from app.models import *
-from app.tasks.forumtasks import importTopicList, checkAllForumAccounts
-from app.tasks.importtasks import importRepoScreenshot, checkZipRelease, check_for_updates
 from app.utils import rank_required, addAuditLog, addNotification, get_system_user
 from . import bp
+from .actions import actions
+from ...models import UserRank, Package, db, PackageState, User, AuditSeverity, NotificationType
 
 
 @bp.route("/admin/", methods=["GET", "POST"])
@@ -38,63 +31,7 @@ def admin_page():
 	if request.method == "POST":
 		action = request.form["action"]
 
-		if action == "delstuckreleases":
+		if action == "restore":
-			PackageRelease.query.filter(PackageRelease.task_id != None).delete()
-			db.session.commit()
-			return redirect(url_for("admin.admin_page"))
-
-		elif action == "checkreleases":
-			releases = PackageRelease.query.filter(PackageRelease.url.like("/uploads/%")).all()
-
-			tasks = []
-			for release in releases:
-				zippath = release.url.replace("/uploads/", app.config["UPLOAD_DIR"])
-				tasks.append(checkZipRelease.s(release.id, zippath))
-
-			result = group(tasks).apply_async()
-
-			while not result.ready():
-				import time
-				time.sleep(0.1)
-
-			return redirect(url_for("todo.view_editor"))
-
-		elif action == "reimportpackages":
-			tasks = []
-			for package in Package.query.filter(Package.state!=PackageState.DELETED).all():
-				release = package.releases.first()
-				if release:
-					zippath = release.url.replace("/uploads/", app.config["UPLOAD_DIR"])
-					tasks.append(checkZipRelease.s(release.id, zippath))
-
-			result = group(tasks).apply_async()
-
-			while not result.ready():
-				import time
-				time.sleep(0.1)
-
-			return redirect(url_for("todo.view_editor"))
-
-		elif action == "importmodlist":
-			task = importTopicList.delay()
-			return redirect(url_for("tasks.check", id=task.id, r=url_for("todo.topics")))
-
-		elif action == "checkusers":
-			task = checkAllForumAccounts.delay()
-			return redirect(url_for("tasks.check", id=task.id, r=url_for("admin.admin_page")))
-
-		elif action == "importscreenshots":
-			packages = Package.query \
-				.filter(Package.state!=PackageState.DELETED) \
-				.outerjoin(PackageScreenshot, Package.id==PackageScreenshot.package_id) \
-				.filter(PackageScreenshot.id==None) \
-				.all()
-			for package in packages:
-				importRepoScreenshot.delay(package.id)
-
-			return redirect(url_for("admin.admin_page"))
-
-		elif action == "restore":
 			package = Package.query.get(request.form["package"])
 			if package is None:
 				flash("Unknown package", "danger")
@@ -103,119 +40,16 @@ def admin_page():
 				db.session.commit()
 				return redirect(url_for("admin.admin_page"))
 
-		elif action == "recalcscores":
+		elif action in actions:
-			for p in Package.query.all():
+			ret = actions[action]["func"]()
-				p.recalcScore()
+			if ret:
-
+				return ret
-			db.session.commit()
-			return redirect(url_for("admin.admin_page"))
-
-		elif action == "cleanuploads":
-			upload_dir = app.config['UPLOAD_DIR']
-
-			(_, _, filenames) = next(os.walk(upload_dir))
-			existing_uploads = set(filenames)
-
-			if len(existing_uploads) != 0:
-				def getURLsFromDB(column):
-					results = db.session.query(column).filter(column != None, column != "").all()
-					return set([os.path.basename(x[0]) for x in results])
-
-				release_urls = getURLsFromDB(PackageRelease.url)
-				screenshot_urls = getURLsFromDB(PackageScreenshot.url)
-
-				db_urls = release_urls.union(screenshot_urls)
-				unreachable = existing_uploads.difference(db_urls)
-
-				import sys
-				print("On Disk: ", existing_uploads, file=sys.stderr)
-				print("In DB: ", db_urls, file=sys.stderr)
-				print("Unreachable: ", unreachable, file=sys.stderr)
-
-				for filename in unreachable:
-					os.remove(os.path.join(upload_dir, filename))
-
-				flash("Deleted " + str(len(unreachable)) + " unreachable uploads", "success")
-			else:
-				flash("No downloads to create", "danger")
-
-			return redirect(url_for("admin.admin_page"))
-
-		elif action == "delmetapackages":
-			query = MetaPackage.query.filter(~MetaPackage.dependencies.any(), ~MetaPackage.packages.any())
-			count = query.count()
-			query.delete(synchronize_session=False)
-			db.session.commit()
-
-			flash("Deleted " + str(count) + " unused meta packages", "success")
-			return redirect(url_for("admin.admin_page"))
-
-		elif action == "delremovedpackages":
-			query = Package.query.filter_by(state=PackageState.DELETED)
-			count = query.count()
-			for pkg in query.all():
-				pkg.review_thread = None
-				db.session.delete(pkg)
-			db.session.commit()
-
-			flash("Deleted {} soft deleted packages packages".format(count), "success")
-			return redirect(url_for("admin.admin_page"))
-
-		elif action == "addupdateconfig":
-			added = 0
-			for pkg in Package.query.filter(Package.repo != None, Package.releases.any(), Package.update_config == None).all():
-				pkg.update_config = PackageUpdateConfig()
-				pkg.update_config.auto_created = True
-
-				release: PackageRelease = pkg.releases.first()
-				if release and release.commit_hash:
-					pkg.update_config.last_commit = release.commit_hash
-
-				db.session.add(pkg.update_config)
-				added += 1
-
-			db.session.commit()
-
-			flash("Added {} update configs".format(added), "success")
-			return redirect(url_for("admin.admin_page"))
-
-		elif action == "runupdateconfig":
-			check_for_updates.delay()
-
-			flash("Started update configs", "success")
-			return redirect(url_for("admin.admin_page"))
-
-		elif action == "remindwip":
-			users = User.query.filter(User.packages.any(or_(Package.state==PackageState.WIP, Package.state==PackageState.CHANGES_NEEDED)))
-			system_user = get_system_user()
-			for user in users:
-				packages = db.session.query(Package.title).filter(
-						Package.author_id==user.id,
-						or_(Package.state==PackageState.WIP, Package.state==PackageState.CHANGES_NEEDED)) \
-					.all()
-
-				# Who needs translations?
-				packages = [pkg[0] for pkg in packages]
-				if len(packages) >= 3:
-					packages[len(packages) - 1] = "and " + packages[len(packages) - 1]
-					packages_list = ", ".join(packages)
-				else:
-					packages_list = "and ".join(packages)
-
-				havent = "haven't" if len(packages) > 1 else "hasn't"
-				if len(packages_list) + 54  > 100:
-					packages_list = packages_list[0:(100-54-1)] + "…"
-
-				addNotification(user, system_user, NotificationType.PACKAGE_APPROVAL,
-					f"Did you forget? {packages_list} {havent} been submitted for review yet",
-						url_for('todo.view_user', username=user.username))
-			db.session.commit()
 
 		else:
 			flash("Unknown action: " + action, "danger")
 
 	deleted_packages = Package.query.filter(Package.state==PackageState.DELETED).all()
-	return render_template("admin/list.html", deleted_packages=deleted_packages)
+	return render_template("admin/list.html", deleted_packages=deleted_packages, actions=actions)
 
 class SwitchUserForm(FlaskForm):
 	username = StringField("Username")
@@ -255,7 +89,7 @@ def send_bulk_notification():
 				"Sent bulk notification", None, None, form.title.data)
 
 		users = User.query.filter(User.rank >= UserRank.NEW_MEMBER).all()
-		addNotification(users, current_user, NotificationType.OTHER, form.title.data, form.url.data, None)
+		addNotification(users, get_system_user(), NotificationType.OTHER, form.title.data, form.url.data, None)
 		db.session.commit()
 
 		return redirect(url_for("admin.admin_page"))

app/templates/admin/list.html

@@ -27,19 +27,11 @@
 					<input type="hidden" name="csrf_token" value="{{ csrf_token() }}" />
 					<div class="row px-3">
 						<select name="action" class="custom-select col">
-							<option value="cleanuploads" selected>Delete unreachable uploads</option>
+							{% for id, action in actions.items() %}
-							<option value="delmetapackages">Delete unused meta packages</option>
+								<option value="{{ id }}" {% if loop.first %}selected{% endif %}>
-							<option value="delstuckreleases">Delete stuck releases</option>
+									{{ action["title"] }}
-							<option value="reimportpackages">Reimport meta</option>
+								</option>
-							<option value="recalcscores">Recalculate package scores</option>
+							{% endfor %}
-							<option value="div">------</option>
-							<option value="checkreleases">Validate all Zip releases</option>
-							<option value="importmodlist">Import forum topics</option>
-							<option value="checkusers">Check forum users</option>
-							<option value="importscreenshots">Import screenshots from VCS</option>
-							<option value="addupdateconfig">Add update configs</option>
-							<option value="runupdateconfig">Run update configs</option>
-							<option value="remindwip">Send WIP/Changes needed notifications</option>
 						</select>
 						<input type="submit" value="Perform" class="col-sm-auto btn btn-primary ml-2" />
 					</div>