contentdb/app/blueprints/admin/actions.py

388 lines
12 KiB
Python
Raw Normal View History

2021-07-30 20:42:11 +02:00
# ContentDB
# Copyright (C) 2018-21 rubenwardy
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <https://www.gnu.org/licenses/>.
import os
2021-07-30 20:50:52 +02:00
from typing import List
2021-07-30 20:42:11 +02:00
2021-07-31 22:03:45 +02:00
import requests
from celery import group, uuid
2023-05-12 02:13:09 +02:00
from flask import redirect, url_for, flash, current_app
2022-01-25 22:37:35 +01:00
from sqlalchemy import or_, and_
2021-07-30 20:42:11 +02:00
2022-01-26 20:12:42 +01:00
from app.models import PackageRelease, db, Package, PackageState, PackageScreenshot, MetaPackage, User, \
2024-01-15 01:48:17 +01:00
NotificationType, PackageUpdateConfig, License, UserRank, PackageType, Thread
2022-02-11 18:17:11 +01:00
from app.tasks.emails import send_pending_digests
2023-06-19 22:27:49 +02:00
from app.tasks.forumtasks import import_topic_list, check_all_forum_accounts
2024-03-03 02:13:05 +01:00
from app.tasks.importtasks import import_repo_screenshot, check_zip_release, check_for_updates, update_all_game_support, \
import_languages
from app.tasks.usertasks import import_github_user_ids
2023-06-19 22:27:49 +02:00
from app.utils import add_notification, get_system_user
2021-07-30 20:42:11 +02:00
actions = {}
2022-01-26 20:12:42 +01:00
2021-07-30 20:42:11 +02:00
def action(title: str):
def func(f):
name = f.__name__
actions[name] = {
"title": title,
"func": f,
}
return f
return func
2022-01-26 20:12:42 +01:00
2021-07-30 20:42:11 +02:00
@action("Delete stuck releases")
2021-07-30 20:50:52 +02:00
def del_stuck_releases():
2022-01-26 20:12:42 +01:00
PackageRelease.query.filter(PackageRelease.task_id.isnot(None)).delete()
2021-07-30 20:42:11 +02:00
db.session.commit()
return redirect(url_for("admin.admin_page"))
2022-01-26 20:12:42 +01:00
@action("Import forum topic list")
2021-07-30 20:42:11 +02:00
def import_topic_list():
2023-06-19 22:27:49 +02:00
task = import_topic_list.delay()
2021-07-30 20:42:11 +02:00
return redirect(url_for("tasks.check", id=task.id, r=url_for("todo.topics")))
2022-01-26 20:12:42 +01:00
2021-07-30 20:42:11 +02:00
@action("Check all forum accounts")
def check_all_forum_accounts():
2023-06-19 22:27:49 +02:00
task = check_all_forum_accounts.delay()
2021-07-30 20:42:11 +02:00
return redirect(url_for("tasks.check", id=task.id, r=url_for("admin.admin_page")))
2022-01-26 20:12:42 +01:00
2023-05-12 02:13:09 +02:00
@action("Delete unused uploads")
2021-07-30 20:50:52 +02:00
def clean_uploads():
2022-01-26 20:12:42 +01:00
upload_dir = current_app.config['UPLOAD_DIR']
2021-07-30 20:42:11 +02:00
(_, _, filenames) = next(os.walk(upload_dir))
existing_uploads = set(filenames)
if len(existing_uploads) != 0:
2022-01-26 20:12:42 +01:00
def get_filenames_from_column(column):
results = db.session.query(column).filter(column.isnot(None), column != "").all()
2021-07-30 20:42:11 +02:00
return set([os.path.basename(x[0]) for x in results])
2022-01-26 20:12:42 +01:00
release_urls = get_filenames_from_column(PackageRelease.url)
screenshot_urls = get_filenames_from_column(PackageScreenshot.url)
pp_urls = get_filenames_from_column(User.profile_pic)
2021-07-30 20:42:11 +02:00
db_urls = release_urls.union(screenshot_urls).union(pp_urls)
2021-07-30 20:42:11 +02:00
unreachable = existing_uploads.difference(db_urls)
import sys
print("On Disk: ", existing_uploads, file=sys.stderr)
print("In DB: ", db_urls, file=sys.stderr)
print("Unreachable: ", unreachable, file=sys.stderr)
for filename in unreachable:
os.remove(os.path.join(upload_dir, filename))
flash("Deleted " + str(len(unreachable)) + " unreachable uploads", "success")
else:
flash("No downloads to create", "danger")
return redirect(url_for("admin.admin_page"))
2022-01-26 20:12:42 +01:00
2022-09-01 23:45:53 +02:00
@action("Delete unused mod names")
def del_mod_names():
2021-07-30 20:42:11 +02:00
query = MetaPackage.query.filter(~MetaPackage.dependencies.any(), ~MetaPackage.packages.any())
count = query.count()
query.delete(synchronize_session=False)
db.session.commit()
2022-09-01 23:45:53 +02:00
flash("Deleted " + str(count) + " unused mod names", "success")
2021-07-30 20:42:11 +02:00
return redirect(url_for("admin.admin_page"))
2022-01-26 20:12:42 +01:00
2021-07-30 20:42:11 +02:00
@action("Run update configs")
2021-07-30 20:50:52 +02:00
def run_update_config():
2021-07-30 20:42:11 +02:00
check_for_updates.delay()
flash("Started update configs", "success")
return redirect(url_for("admin.admin_page"))
2022-01-26 20:12:42 +01:00
2021-07-30 20:50:52 +02:00
def _package_list(packages: List[str]):
# Who needs translations?
if len(packages) >= 3:
packages[len(packages) - 1] = "and " + packages[len(packages) - 1]
return ", ".join(packages)
2021-07-30 20:50:52 +02:00
else:
return " and ".join(packages)
2021-07-30 20:50:52 +02:00
2022-01-26 20:12:42 +01:00
2021-07-30 20:42:11 +02:00
@action("Send WIP package notification")
2021-07-30 20:50:52 +02:00
def remind_wip():
2023-06-19 20:32:36 +02:00
users = User.query.filter(User.packages.any(or_(
Package.state == PackageState.WIP, Package.state == PackageState.CHANGES_NEEDED)))
2021-07-30 20:42:11 +02:00
system_user = get_system_user()
for user in users:
packages = Package.query.filter(
2022-01-26 20:12:42 +01:00
Package.author_id == user.id,
2023-06-19 20:32:36 +02:00
or_(Package.state == PackageState.WIP, Package.state == PackageState.CHANGES_NEEDED)) \
2021-07-30 20:42:11 +02:00
.all()
packages = [pkg.title for pkg in packages]
2021-07-30 20:50:52 +02:00
packages_list = _package_list(packages)
2021-07-30 20:42:11 +02:00
havent = "haven't" if len(packages) > 1 else "hasn't"
2023-06-19 22:27:49 +02:00
add_notification(user, system_user, NotificationType.PACKAGE_APPROVAL,
2021-07-30 20:42:11 +02:00
f"Did you forget? {packages_list} {havent} been submitted for review yet",
2023-06-19 22:27:49 +02:00
url_for('todo.view_user', username=user.username))
2021-07-30 20:42:11 +02:00
db.session.commit()
2021-07-30 20:50:52 +02:00
2022-01-26 20:12:42 +01:00
2021-07-30 20:50:52 +02:00
@action("Send outdated package notification")
def remind_outdated():
users = User.query.filter(User.maintained_packages.any(
Package.update_config.has(PackageUpdateConfig.outdated_at.isnot(None))))
system_user = get_system_user()
for user in users:
packages = Package.query.filter(
Package.maintainers.contains(user),
2021-07-30 20:50:52 +02:00
Package.update_config.has(PackageUpdateConfig.outdated_at.isnot(None))) \
.all()
packages = [pkg.title for pkg in packages]
2021-07-30 20:50:52 +02:00
packages_list = _package_list(packages)
2023-06-19 22:27:49 +02:00
add_notification(user, system_user, NotificationType.PACKAGE_APPROVAL,
2021-07-30 20:50:52 +02:00
f"The following packages may be outdated: {packages_list}",
2023-06-19 22:27:49 +02:00
url_for('todo.view_user', username=user.username))
2021-07-30 20:50:52 +02:00
db.session.commit()
2021-07-31 22:03:45 +02:00
2022-01-26 20:12:42 +01:00
2021-07-31 22:03:45 +02:00
@action("Import licenses from SPDX")
def import_licenses():
renames = {
"GPLv2": "GPL-2.0-only",
"GPLv3": "GPL-3.0-only",
"AGPLv2": "AGPL-2.0-only",
"AGPLv3": "AGPL-3.0-only",
"LGPLv2.1": "LGPL-2.1-only",
"LGPLv3": "LGPL-3.0-only",
"Apache 2.0": "Apache-2.0",
2021-07-31 22:03:45 +02:00
"BSD 2-Clause / FreeBSD": "BSD-2-Clause-FreeBSD",
"BSD 3-Clause": "BSD-3-Clause",
2021-07-31 22:03:45 +02:00
"CC0": "CC0-1.0",
"CC BY 3.0": "CC-BY-3.0",
"CC BY 4.0": "CC-BY-4.0",
"CC BY-NC-SA 3.0": "CC-BY-NC-SA-3.0",
"CC BY-SA 3.0": "CC-BY-SA-3.0",
"CC BY-SA 4.0": "CC-BY-SA-4.0",
"NPOSLv3": "NPOSL-3.0",
"MPL 2.0": "MPL-2.0",
"EUPLv1.2": "EUPL-1.2",
"SIL Open Font License v1.1": "OFL-1.1",
}
for old_name, new_name in renames.items():
License.query.filter_by(name=old_name).update({ "name": new_name })
r = requests.get(
"https://raw.githubusercontent.com/spdx/license-list-data/master/json/licenses.json")
licenses = r.json()["licenses"]
existing_licenses = {}
2023-06-19 20:32:36 +02:00
for license_data in License.query.all():
assert license_data.name not in renames.keys()
existing_licenses[license_data.name.lower()] = license_data
2021-07-31 22:03:45 +02:00
2023-06-19 20:32:36 +02:00
for license_data in licenses:
obj = existing_licenses.get(license_data["licenseId"].lower())
2021-07-31 22:03:45 +02:00
if obj:
2023-06-19 20:32:36 +02:00
obj.url = license_data["reference"]
elif license_data.get("isOsiApproved") and license_data.get("isFsfLibre") and not license_data["isDeprecatedLicenseId"]:
obj = License(license_data["licenseId"], True, license_data["reference"])
2021-07-31 22:03:45 +02:00
db.session.add(obj)
db.session.commit()
@action("Delete inactive users")
def delete_inactive_users():
2022-02-13 10:53:12 +01:00
users = User.query.filter(User.is_active == False, ~User.packages.any(), ~User.forum_topics.any(),
2022-01-26 20:12:42 +01:00
User.rank == UserRank.NOT_JOINED).all()
for user in users:
db.session.delete(user)
db.session.commit()
2022-01-25 22:37:35 +01:00
@action("Send Video URL notification")
def remind_video_url():
users = User.query.filter(User.maintained_packages.any(
2023-06-19 20:32:36 +02:00
and_(Package.video_url == None, Package.type == PackageType.GAME, Package.state == PackageState.APPROVED)))
2022-01-25 22:37:35 +01:00
system_user = get_system_user()
for user in users:
packages = Package.query.filter(
2023-06-19 20:32:36 +02:00
or_(Package.author == user, Package.maintainers.contains(user)),
Package.video_url == None,
2022-01-25 22:37:35 +01:00
Package.type == PackageType.GAME,
Package.state == PackageState.APPROVED) \
.all()
package_names = [pkg.title for pkg in packages]
packages_list = _package_list(package_names)
2022-01-25 22:37:35 +01:00
2023-06-19 22:27:49 +02:00
add_notification(user, system_user, NotificationType.PACKAGE_APPROVAL,
2022-01-25 22:37:35 +01:00
f"You should add a video to {packages_list}",
2023-06-19 22:27:49 +02:00
url_for('users.profile', username=user.username))
2022-01-25 22:37:35 +01:00
db.session.commit()
2022-01-26 03:51:40 +01:00
@action("Send missing game support notifications")
def remind_missing_game_support():
users = User.query.filter(
User.maintained_packages.any(and_(
Package.state != PackageState.DELETED,
Package.type.in_([PackageType.MOD, PackageType.TXP]),
~Package.supported_games.any(),
Package.supports_all_games == False))).all()
system_user = get_system_user()
for user in users:
packages = Package.query.filter(
Package.maintainers.contains(user),
Package.state != PackageState.DELETED,
Package.type.in_([PackageType.MOD, PackageType.TXP]),
~Package.supported_games.any(),
Package.supports_all_games == False) \
.all()
packages = [pkg.title for pkg in packages]
packages_list = _package_list(packages)
2023-06-19 22:27:49 +02:00
add_notification(user, system_user, NotificationType.PACKAGE_APPROVAL,
f"You need to confirm whether the following packages support all games: {packages_list}",
url_for('todo.all_game_support', username=user.username))
db.session.commit()
@action("Detect game support")
def detect_game_support():
task_id = uuid()
2023-06-19 22:27:49 +02:00
update_all_game_support.apply_async((), task_id=task_id)
return redirect(url_for("tasks.check", id=task_id, r=url_for("admin.admin_page")))
2022-02-11 18:17:11 +01:00
@action("Send pending notif digests")
def do_send_pending_digests():
send_pending_digests.delay()
2022-08-23 03:24:12 +02:00
@action("Import user ids from GitHub")
def do_import_github_user_ids():
task_id = uuid()
import_github_user_ids.apply_async((), task_id=task_id)
return redirect(url_for("tasks.check", id=task_id, r=url_for("admin.admin_page")))
2023-05-12 02:13:09 +02:00
@action("DANGER: Delete removed packages")
def del_removed_packages():
query = Package.query.filter_by(state=PackageState.DELETED)
count = query.count()
for pkg in query.all():
pkg.review_thread = None
db.session.delete(pkg)
db.session.commit()
flash("Deleted {} soft deleted packages packages".format(count), "success")
return redirect(url_for("admin.admin_page"))
2023-05-12 02:13:09 +02:00
@action("DANGER: Check all releases (postReleaseCheckUpdate)")
def check_releases():
releases = PackageRelease.query.filter(PackageRelease.url.like("/uploads/%")).all()
tasks = []
for release in releases:
2023-06-19 22:27:49 +02:00
tasks.append(check_zip_release.s(release.id, release.file_path))
2023-05-12 02:13:09 +02:00
result = group(tasks).apply_async()
while not result.ready():
import time
time.sleep(0.1)
2023-05-12 02:13:09 +02:00
return redirect(url_for("todo.view_editor"))
@action("DANGER: Check latest release of all packages (postReleaseCheckUpdate)")
def reimport_packages():
tasks = []
2024-03-03 02:13:05 +01:00
for package in Package.query.filter(Package.state == PackageState.APPROVED).all():
2023-05-12 02:13:09 +02:00
release = package.releases.first()
if release:
2023-06-19 22:27:49 +02:00
tasks.append(check_zip_release.s(release.id, release.file_path))
2023-05-12 02:13:09 +02:00
result = group(tasks).apply_async()
while not result.ready():
import time
time.sleep(0.1)
return redirect(url_for("todo.view_editor"))
2024-03-03 02:13:05 +01:00
@action("DANGER: Import translations")
def reimport_translations():
tasks = []
for package in Package.query.filter(Package.state == PackageState.APPROVED).all():
release = package.releases.first()
if release:
tasks.append(import_languages.s(release.id, release.file_path))
result = group(tasks).apply_async()
while not result.ready():
import time
time.sleep(0.1)
return redirect(url_for("todo.view_editor"))
2023-05-12 02:13:09 +02:00
@action("DANGER: Import screenshots from Git")
def import_screenshots():
packages = Package.query \
.filter(Package.state != PackageState.DELETED) \
.outerjoin(PackageScreenshot, Package.id == PackageScreenshot.package_id) \
2023-06-19 20:32:36 +02:00
.filter(PackageScreenshot.id == None) \
2023-05-12 02:13:09 +02:00
.all()
for package in packages:
2023-06-19 22:27:49 +02:00
import_repo_screenshot.delay(package.id)
2023-05-12 02:13:09 +02:00
return redirect(url_for("admin.admin_page"))
2024-01-15 01:48:17 +01:00
@action("DANGER: Delete empty threads")
def delete_empty_threads():
query = Thread.query.filter(~Thread.replies.any())
count = query.count()
for thread in query.all():
thread.watchers.clear()
db.session.delete(thread)
db.session.commit()
flash(f"Deleted {count} threads", "success")
return redirect(url_for("admin.admin_page"))