diff --git a/app/__init__.py b/app/__init__.py index a2b12933..f09d6528 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -15,16 +15,18 @@ # along with this program. If not, see . import datetime +import os +import redis -from flask import * -from flask_gravatar import Gravatar -from flask_mail import Mail -from flask_github import GitHub -from flask_wtf.csrf import CSRFProtect -from flask_flatpages import FlatPages +from flask import redirect, url_for, render_template, flash, request, Flask, send_from_directory, make_response from flask_babel import Babel, gettext +from flask_flatpages import FlatPages +from flask_github import GitHub +from flask_gravatar import Gravatar from flask_login import logout_user, current_user, LoginManager -import os, redis +from flask_mail import Mail +from flask_wtf.csrf import CSRFProtect + from app.markdown import init_markdown, MARKDOWN_EXTENSIONS, MARKDOWN_EXTENSION_CONFIG app = Flask(__name__, static_folder="public/static") diff --git a/app/blueprints/__init__.py b/app/blueprints/__init__.py index 74aa9ae3..c39f6795 100644 --- a/app/blueprints/__init__.py +++ b/app/blueprints/__init__.py @@ -1,4 +1,22 @@ -import os, importlib +# ContentDB +# Copyright (C) 2018-21 rubenwardy +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +import importlib +import os + def create_blueprints(app): dir = os.path.dirname(os.path.realpath(__file__)) diff --git a/app/blueprints/admin/actions.py b/app/blueprints/admin/actions.py index e78345a0..ef62f1a1 100644 --- a/app/blueprints/admin/actions.py +++ b/app/blueprints/admin/actions.py @@ -129,12 +129,13 @@ def _package_list(packages: List[str]): @action("Send WIP package notification") def remind_wip(): - users = User.query.filter(User.packages.any(or_(Package.state == PackageState.WIP, Package.state == PackageState.CHANGES_NEEDED))) + users = User.query.filter(User.packages.any(or_( + Package.state == PackageState.WIP, Package.state == PackageState.CHANGES_NEEDED))) system_user = get_system_user() for user in users: packages = db.session.query(Package.title).filter( Package.author_id == user.id, - or_(Package.state == PackageState.WIP, Package.state==PackageState.CHANGES_NEEDED)) \ + or_(Package.state == PackageState.WIP, Package.state == PackageState.CHANGES_NEEDED)) \ .all() packages = [pkg[0] for pkg in packages] @@ -200,17 +201,16 @@ def import_licenses(): licenses = r.json()["licenses"] existing_licenses = {} - for license in License.query.all(): - assert license.name not in renames.keys() - existing_licenses[license.name.lower()] = license + for license_data in License.query.all(): + assert license_data.name not in renames.keys() + existing_licenses[license_data.name.lower()] = license_data - for license in licenses: - obj = existing_licenses.get(license["licenseId"].lower()) + for license_data in licenses: + obj = existing_licenses.get(license_data["licenseId"].lower()) if obj: - obj.url = license["reference"] - elif license.get("isOsiApproved") and license.get("isFsfLibre") and \ - not license["isDeprecatedLicenseId"]: - obj = License(license["licenseId"], True, license["reference"]) + obj.url = license_data["reference"] + elif license_data.get("isOsiApproved") and license_data.get("isFsfLibre") and not license_data["isDeprecatedLicenseId"]: + obj = License(license_data["licenseId"], True, license_data["reference"]) db.session.add(obj) db.session.commit() @@ -228,12 +228,12 @@ def delete_inactive_users(): @action("Send Video URL notification") def remind_video_url(): users = User.query.filter(User.maintained_packages.any( - and_(Package.video_url==None, Package.type==PackageType.GAME, Package.state==PackageState.APPROVED))) + and_(Package.video_url == None, Package.type == PackageType.GAME, Package.state == PackageState.APPROVED))) system_user = get_system_user() for user in users: packages = db.session.query(Package.title).filter( - or_(Package.author==user, Package.maintainers.contains(user)), - Package.video_url==None, + or_(Package.author == user, Package.maintainers.contains(user)), + Package.video_url == None, Package.type == PackageType.GAME, Package.state == PackageState.APPROVED) \ .all() @@ -341,7 +341,7 @@ def import_screenshots(): packages = Package.query \ .filter(Package.state != PackageState.DELETED) \ .outerjoin(PackageScreenshot, Package.id == PackageScreenshot.package_id) \ - .filter(PackageScreenshot.id==None) \ + .filter(PackageScreenshot.id == None) \ .all() for package in packages: importRepoScreenshot.delay(package.id) diff --git a/app/blueprints/admin/admin.py b/app/blueprints/admin/admin.py index 5c27cee4..8313e80a 100644 --- a/app/blueprints/admin/admin.py +++ b/app/blueprints/admin/admin.py @@ -22,7 +22,7 @@ from wtforms.validators import InputRequired, Length from app.utils import rank_required, addAuditLog, addNotification, get_system_user from . import bp from .actions import actions -from ...models import UserRank, Package, db, PackageState, User, AuditSeverity, NotificationType +from app.models import UserRank, Package, db, PackageState, User, AuditSeverity, NotificationType @bp.route("/admin/", methods=["GET", "POST"]) diff --git a/app/blueprints/admin/email.py b/app/blueprints/admin/email.py index 6899debc..fbfe2611 100644 --- a/app/blueprints/admin/email.py +++ b/app/blueprints/admin/email.py @@ -24,7 +24,7 @@ from app.markdown import render_markdown from app.tasks.emails import send_user_email, send_bulk_email as task_send_bulk from app.utils import rank_required, addAuditLog from . import bp -from ...models import UserRank, User, AuditSeverity +from app.models import UserRank, User, AuditSeverity class SendEmailForm(FlaskForm): @@ -54,7 +54,7 @@ def send_single_email(): text = form.text.data html = render_markdown(text) - task = send_user_email.delay(user.email, user.locale or "en",form.subject.data, text, html) + task = send_user_email.delay(user.email, user.locale or "en", form.subject.data, text, html) return redirect(url_for("tasks.check", id=task.id, r=next_url)) return render_template("admin/send_email.html", form=form, user=user) diff --git a/app/blueprints/admin/licenseseditor.py b/app/blueprints/admin/licenseseditor.py index 9dfa8802..06544225 100644 --- a/app/blueprints/admin/licenseseditor.py +++ b/app/blueprints/admin/licenseseditor.py @@ -23,7 +23,7 @@ from wtforms.validators import InputRequired, Length, Optional from app.utils import rank_required, nonEmptyOrNone, addAuditLog from . import bp -from ...models import UserRank, License, db, AuditSeverity +from app.models import UserRank, License, db, AuditSeverity @bp.route("/licenses/") diff --git a/app/blueprints/admin/tagseditor.py b/app/blueprints/admin/tagseditor.py index 228a991e..5a62bddd 100644 --- a/app/blueprints/admin/tagseditor.py +++ b/app/blueprints/admin/tagseditor.py @@ -22,8 +22,8 @@ from wtforms import StringField, TextAreaField, BooleanField, SubmitField from wtforms.validators import InputRequired, Length, Optional, Regexp from . import bp -from ...models import Permission, Tag, db, AuditSeverity -from ...utils import addAuditLog +from app.models import Permission, Tag, db, AuditSeverity +from app.utils import addAuditLog @bp.route("/tags/") @@ -45,7 +45,8 @@ def tag_list(): class TagForm(FlaskForm): title = StringField("Title", [InputRequired(), Length(3, 100)]) description = TextAreaField("Description", [Optional(), Length(0, 500)]) - name = StringField("Name", [Optional(), Length(1, 20), Regexp("^[a-z0-9_]", 0, "Lower case letters (a-z), digits (0-9), and underscores (_) only")]) + name = StringField("Name", [Optional(), Length(1, 20), Regexp("^[a-z0-9_]", 0, + "Lower case letters (a-z), digits (0-9), and underscores (_) only")]) is_protected = BooleanField("Is Protected") submit = SubmitField("Save") @@ -63,7 +64,7 @@ def create_edit_tag(name=None): if not Permission.check_perm(current_user, Permission.EDIT_TAGS if tag else Permission.CREATE_TAG): abort(403) - form = TagForm( obj=tag) + form = TagForm(obj=tag) if form.validate_on_submit(): if tag is None: tag = Tag(form.title.data) diff --git a/app/blueprints/admin/versioneditor.py b/app/blueprints/admin/versioneditor.py index 92381345..75ee92f6 100644 --- a/app/blueprints/admin/versioneditor.py +++ b/app/blueprints/admin/versioneditor.py @@ -23,13 +23,14 @@ from wtforms.validators import InputRequired, Length from app.utils import rank_required, addAuditLog from . import bp -from ...models import UserRank, MinetestRelease, db, AuditSeverity +from app.models import UserRank, MinetestRelease, db, AuditSeverity @bp.route("/versions/") @rank_required(UserRank.MODERATOR) def version_list(): - return render_template("admin/versions/list.html", versions=MinetestRelease.query.order_by(db.asc(MinetestRelease.id)).all()) + return render_template("admin/versions/list.html", + versions=MinetestRelease.query.order_by(db.asc(MinetestRelease.id)).all()) class VersionForm(FlaskForm): diff --git a/app/blueprints/admin/warningseditor.py b/app/blueprints/admin/warningseditor.py index 94a8df62..397c661e 100644 --- a/app/blueprints/admin/warningseditor.py +++ b/app/blueprints/admin/warningseditor.py @@ -15,14 +15,14 @@ # along with this program. If not, see . -from flask import redirect, render_template, abort, url_for, request, flash +from flask import redirect, render_template, abort, url_for, request from flask_wtf import FlaskForm from wtforms import StringField, TextAreaField, SubmitField from wtforms.validators import InputRequired, Length, Optional, Regexp from app.utils import rank_required from . import bp -from ...models import UserRank, ContentWarning, db +from app.models import UserRank, ContentWarning, db @bp.route("/admin/warnings/") diff --git a/app/blueprints/api/endpoints.py b/app/blueprints/api/endpoints.py index 7238e8e9..d8211db2 100644 --- a/app/blueprints/api/endpoints.py +++ b/app/blueprints/api/endpoints.py @@ -35,7 +35,7 @@ from . import bp from .auth import is_api_authd from .support import error, api_create_vcs_release, api_create_zip_release, api_create_screenshot, \ api_order_screenshots, api_edit_package, api_set_cover_image -from ...utils.minetest_hypertext import html_to_minetest +from app.utils.minetest_hypertext import html_to_minetest def cors_allowed(f): @@ -65,15 +65,15 @@ def cached(max_age: int): @cors_allowed @cached(300) def packages(): - qb = QueryBuilder(request.args) + qb = QueryBuilder(request.args) query = qb.buildPackageQuery() if request.args.get("fmt") == "keys": - return jsonify([package.as_key_dict() for package in query.all()]) + return jsonify([pkg.as_key_dict() for pkg in query.all()]) pkgs = qb.convertToDictionary(query.all()) if "engine_version" in request.args or "protocol_version" in request.args: - pkgs = [package for package in pkgs if package.get("release")] + pkgs = [pkg for pkg in pkgs if pkg.get("release")] # Promote featured packages if "sort" not in request.args and "order" not in request.args and "q" not in request.args: @@ -92,7 +92,7 @@ def packages(): @bp.route("/api/packages///") @is_package_page @cors_allowed -def package(package): +def package_view(package): return jsonify(package.as_dict(current_app.config["BASE_URL"])) @@ -119,12 +119,12 @@ def edit_package(token, package): def resolve_package_deps(out, package, only_hard, depth=1): - id = package.get_id() - if id in out: + id_ = package.get_id() + if id_ in out: return ret = [] - out[id] = ret + out[id_] = ret if package.type != PackageType.MOD: return @@ -173,8 +173,8 @@ def package_dependencies(package): @bp.route("/api/topics/") @cors_allowed def topics(): - qb = QueryBuilder(request.args) - query = qb.buildTopicQuery(show_added=True) + qb = QueryBuilder(request.args) + query = qb.buildTopicQuery(show_added=True) return jsonify([t.as_dict() for t in query.all()]) @@ -285,8 +285,8 @@ def create_release(token, package): @bp.route("/api/packages///releases//") @is_package_page @cors_allowed -def release(package: Package, id: int): - release = PackageRelease.query.get(id) +def release_view(package: Package, id_: int): + release = PackageRelease.query.get(id_) if release is None or release.package != package: error(404, "Release not found") @@ -298,15 +298,15 @@ def release(package: Package, id: int): @is_package_page @is_api_authd @cors_allowed -def delete_release(token: APIToken, package: Package, id: int): - release = PackageRelease.query.get(id) +def delete_release(token: APIToken, package: Package, id_: int): + release = PackageRelease.query.get(id_) if release is None or release.package != package: error(404, "Release not found") if not token: error(401, "Authentication needed") - if not token.canOperateOnPackage(package): + if not token.can_operate_on_package(package): error(403, "API token does not have access to the package") if not release.check_perm(token.owner, Permission.DELETE_RELEASE): @@ -352,8 +352,8 @@ def create_screenshot(token: APIToken, package: Package): @bp.route("/api/packages///screenshots//") @is_package_page @cors_allowed -def screenshot(package, id): - ss = PackageScreenshot.query.get(id) +def screenshot(package, id_): + ss = PackageScreenshot.query.get(id_) if ss is None or ss.package != package: error(404, "Screenshot not found") @@ -365,8 +365,8 @@ def screenshot(package, id): @is_package_page @is_api_authd @cors_allowed -def delete_screenshot(token: APIToken, package: Package, id: int): - ss = PackageScreenshot.query.get(id) +def delete_screenshot(token: APIToken, package: Package, id_: int): + ss = PackageScreenshot.query.get(id_) if ss is None or ss.package != package: error(404, "Screenshot not found") @@ -376,7 +376,7 @@ def delete_screenshot(token: APIToken, package: Package, id: int): if not package.check_perm(token.owner, Permission.ADD_SCREENSHOTS): error(403, "You do not have the permission to delete screenshots") - if not token.canOperateOnPackage(package): + if not token.can_operate_on_package(package): error(403, "API token does not have access to the package") if package.cover_image == ss: @@ -401,7 +401,7 @@ def order_screenshots(token: APIToken, package: Package): if not package.check_perm(token.owner, Permission.ADD_SCREENSHOTS): error(403, "You do not have the permission to change screenshots") - if not token.canOperateOnPackage(package): + if not token.can_operate_on_package(package): error(403, "API token does not have access to the package") json = request.json @@ -423,7 +423,7 @@ def set_cover_image(token: APIToken, package: Package): if not package.check_perm(token.owner, Permission.ADD_SCREENSHOTS): error(403, "You do not have the permission to change screenshots") - if not token.canOperateOnPackage(package): + if not token.can_operate_on_package(package): error(403, "API token does not have access to the package") json = request.json @@ -498,7 +498,7 @@ def all_package_stats(): @cors_allowed @cached(300) def package_scores(): - qb = QueryBuilder(request.args) + qb = QueryBuilder(request.args) query = qb.buildPackageQuery() pkgs = [package.as_score_dict() for package in query.all()] @@ -520,19 +520,19 @@ def content_warnings(): @bp.route("/api/licenses/") @cors_allowed def licenses(): - return jsonify([ { "name": license.name, "is_foss": license.is_foss } \ - for license in License.query.order_by(db.asc(License.name)).all() ]) + all_licenses = License.query.order_by(db.asc(License.name)).all() + return jsonify([{"name": license.name, "is_foss": license.is_foss} for license in all_licenses]) @bp.route("/api/homepage/") @cors_allowed def homepage(): - query = Package.query.filter_by(state=PackageState.APPROVED) - count = query.count() + query = Package.query.filter_by(state=PackageState.APPROVED) + count = query.count() spotlight = query.filter(Package.tags.any(name="spotlight")).order_by( func.random()).limit(6).all() - new = query.order_by(db.desc(Package.approved_at)).limit(4).all() + new = query.order_by(db.desc(Package.approved_at)).limit(4).all() pop_mod = query.filter_by(type=PackageType.MOD).order_by(db.desc(Package.score)).limit(8).all() pop_gam = query.filter_by(type=PackageType.GAME).order_by(db.desc(Package.score)).limit(8).all() pop_txp = query.filter_by(type=PackageType.TXP).order_by(db.desc(Package.score)).limit(8).all() @@ -548,19 +548,19 @@ def homepage(): downloads_result = db.session.query(func.sum(Package.downloads)).one_or_none() downloads = 0 if not downloads_result or not downloads_result[0] else downloads_result[0] - def mapPackages(packages: List[Package]): + def map_packages(packages: List[Package]): return [pkg.as_short_dict(current_app.config["BASE_URL"]) for pkg in packages] return jsonify({ "count": count, "downloads": downloads, - "spotlight": mapPackages(spotlight), - "new": mapPackages(new), - "updated": mapPackages(updated), - "pop_mod": mapPackages(pop_mod), - "pop_txp": mapPackages(pop_txp), - "pop_game": mapPackages(pop_gam), - "high_reviewed": mapPackages(high_reviewed) + "spotlight": map_packages(spotlight), + "new": map_packages(new), + "updated": map_packages(updated), + "pop_mod": map_packages(pop_mod), + "pop_txp": map_packages(pop_txp), + "pop_game": map_packages(pop_gam), + "high_reviewed": map_packages(high_reviewed) }) diff --git a/app/blueprints/api/support.py b/app/blueprints/api/support.py index 035574e5..7f0d9550 100644 --- a/app/blueprints/api/support.py +++ b/app/blueprints/api/support.py @@ -26,6 +26,7 @@ from app.models import APIToken, Package, MinetestRelease, PackageScreenshot def error(code: int, msg: str): abort(make_response(jsonify({ "success": False, "error": msg }), code)) + # Catches LogicErrors and aborts with JSON error def guard(f): def ret(*args, **kwargs): @@ -39,7 +40,7 @@ def guard(f): def api_create_vcs_release(token: APIToken, package: Package, title: str, ref: str, min_v: MinetestRelease = None, max_v: MinetestRelease = None, reason="API"): - if not token.canOperateOnPackage(package): + if not token.can_operate_on_package(package): error(403, "API token does not have access to the package") reason += ", token=" + token.name @@ -54,8 +55,8 @@ def api_create_vcs_release(token: APIToken, package: Package, title: str, ref: s def api_create_zip_release(token: APIToken, package: Package, title: str, file, - min_v: MinetestRelease = None, max_v: MinetestRelease = None, reason="API", commit_hash:str=None): - if not token.canOperateOnPackage(package): + min_v: MinetestRelease = None, max_v: MinetestRelease = None, reason="API", commit_hash: str = None): + if not token.can_operate_on_package(package): error(403, "API token does not have access to the package") reason += ", token=" + token.name @@ -70,7 +71,7 @@ def api_create_zip_release(token: APIToken, package: Package, title: str, file, def api_create_screenshot(token: APIToken, package: Package, title: str, file, is_cover_image: bool, reason="API"): - if not token.canOperateOnPackage(package): + if not token.can_operate_on_package(package): error(403, "API token does not have access to the package") reason += ", token=" + token.name @@ -84,7 +85,7 @@ def api_create_screenshot(token: APIToken, package: Package, title: str, file, i def api_order_screenshots(token: APIToken, package: Package, order: [any]): - if not token.canOperateOnPackage(package): + if not token.can_operate_on_package(package): error(403, "API token does not have access to the package") guard(do_order_screenshots)(token.owner, package, order) @@ -95,7 +96,7 @@ def api_order_screenshots(token: APIToken, package: Package, order: [any]): def api_set_cover_image(token: APIToken, package: Package, cover_image): - if not token.canOperateOnPackage(package): + if not token.can_operate_on_package(package): error(403, "API token does not have access to the package") guard(do_set_cover_image)(token.owner, package, cover_image) @@ -106,7 +107,7 @@ def api_set_cover_image(token: APIToken, package: Package, cover_image): def api_edit_package(token: APIToken, package: Package, data: dict, reason: str = "API"): - if not token.canOperateOnPackage(package): + if not token.can_operate_on_package(package): error(403, "API token does not have access to the package") reason += ", token=" + token.name diff --git a/app/blueprints/api/tokens.py b/app/blueprints/api/tokens.py index 62976109..c9d6690d 100644 --- a/app/blueprints/api/tokens.py +++ b/app/blueprints/api/tokens.py @@ -19,8 +19,8 @@ from flask import render_template, redirect, request, session, url_for, abort from flask_babel import lazy_gettext from flask_login import login_required, current_user from flask_wtf import FlaskForm -from wtforms import * -from wtforms.validators import * +from wtforms import StringField, SubmitField +from wtforms.validators import InputRequired, Length from wtforms_sqlalchemy.fields import QuerySelectField from app.models import db, User, APIToken, Permission diff --git a/app/blueprints/donate/__init__.py b/app/blueprints/donate/__init__.py index 516fdd97..64355d01 100644 --- a/app/blueprints/donate/__init__.py +++ b/app/blueprints/donate/__init__.py @@ -1,3 +1,20 @@ +# ContentDB +# Copyright (C) 2023 rubenwardy +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + + from flask import Blueprint, render_template from flask_login import current_user from sqlalchemy import or_, and_ diff --git a/app/blueprints/gitlab/__init__.py b/app/blueprints/gitlab/__init__.py index 0b6380bb..30edbed9 100644 --- a/app/blueprints/gitlab/__init__.py +++ b/app/blueprints/gitlab/__init__.py @@ -65,7 +65,7 @@ def webhook_impl(): title = ref.replace("refs/tags/", "") else: return error(400, "Unsupported event: '{}'. Only 'push', 'create:tag', and 'ping' are supported." - .format(event or "null")) + .format(event or "null")) # # Perform release diff --git a/app/blueprints/homepage/__init__.py b/app/blueprints/homepage/__init__.py index 5f5fa19f..381f25f7 100644 --- a/app/blueprints/homepage/__init__.py +++ b/app/blueprints/homepage/__init__.py @@ -1,8 +1,25 @@ +# ContentDB +# Copyright (C) 2018-23 rubenwardy +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + from flask import Blueprint, render_template, redirect +from app.models import Package, PackageReview, Thread, User, PackageState, db, PackageType, PackageRelease, Tags, Tag + bp = Blueprint("homepage", __name__) -from app.models import * from sqlalchemy.orm import joinedload, subqueryload from sqlalchemy.sql.expression import func @@ -29,12 +46,12 @@ def home(): joinedload(PackageReview.package).joinedload(Package.author).load_only(User.username, User.display_name), joinedload(PackageReview.package).load_only(Package.title, Package.name).subqueryload(Package.main_screenshot)) - query = Package.query.filter_by(state=PackageState.APPROVED) - count = query.count() + query = Package.query.filter_by(state=PackageState.APPROVED) + count = query.count() spotlight_pkgs = query.filter(Package.tags.any(name="spotlight")).order_by(func.random()).limit(6).all() - new = package_load(query.order_by(db.desc(Package.approved_at))).limit(4).all() + new = package_load(query.order_by(db.desc(Package.approved_at))).limit(4).all() pop_mod = package_load(query.filter_by(type=PackageType.MOD).order_by(db.desc(Package.score))).limit(8).all() pop_gam = package_load(query.filter_by(type=PackageType.GAME).order_by(db.desc(Package.score))).limit(8).all() pop_txp = package_load(query.filter_by(type=PackageType.TXP).order_by(db.desc(Package.score))).limit(8).all() @@ -58,4 +75,5 @@ def home(): .group_by(Tag.id).order_by(db.asc(Tag.title)).all() return render_template("index.html", count=count, downloads=downloads, tags=tags, spotlight_pkgs=spotlight_pkgs, - new=new, updated=updated, pop_mod=pop_mod, pop_txp=pop_txp, pop_gam=pop_gam, high_reviewed=high_reviewed, reviews=reviews) + new=new, updated=updated, pop_mod=pop_mod, pop_txp=pop_txp, pop_gam=pop_gam, high_reviewed=high_reviewed, + reviews=reviews) diff --git a/app/blueprints/metrics/__init__.py b/app/blueprints/metrics/__init__.py index 379692b6..4760e5d4 100644 --- a/app/blueprints/metrics/__init__.py +++ b/app/blueprints/metrics/__init__.py @@ -21,6 +21,7 @@ from app.models import Package, db, User, UserRank, PackageState bp = Blueprint("metrics", __name__) + def generate_metrics(full=False): def write_single_stat(name, help, type, value): fmt = "# HELP {name} {help}\n# TYPE {name} {type}\n{name} {value}\n\n" @@ -31,7 +32,6 @@ def generate_metrics(full=False): pieces = [key + "=" + str(val) for key, val in labels.items()] return ",".join(pieces) - def write_array_stat(name, help, type, data): ret = "# HELP {name} {help}\n# TYPE {name} {type}\n" \ .format(name=name, help=help, type=type) @@ -67,6 +67,7 @@ def generate_metrics(full=False): return ret + @bp.route("/metrics") def metrics(): response = make_response(generate_metrics(), 200) diff --git a/app/blueprints/modnames/__init__.py b/app/blueprints/modnames/__init__.py index 62571e28..6697e104 100644 --- a/app/blueprints/modnames/__init__.py +++ b/app/blueprints/modnames/__init__.py @@ -14,8 +14,7 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . - -from flask import * +from flask import Blueprint, redirect, render_template, abort from sqlalchemy import func from app.models import MetaPackage, Package, db, Dependency, PackageState, ForumTopic diff --git a/app/blueprints/notifications/__init__.py b/app/blueprints/notifications/__init__.py index 03155660..f60f6932 100644 --- a/app/blueprints/notifications/__init__.py +++ b/app/blueprints/notifications/__init__.py @@ -14,7 +14,6 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . - from flask import Blueprint, render_template, redirect, url_for from flask_login import current_user, login_required from sqlalchemy import or_, desc diff --git a/app/blueprints/packages/game_hub.py b/app/blueprints/packages/game_hub.py index f68abc93..d5997a94 100644 --- a/app/blueprints/packages/game_hub.py +++ b/app/blueprints/packages/game_hub.py @@ -19,7 +19,7 @@ from sqlalchemy.orm import joinedload from . import bp from app.utils import is_package_page -from ...models import Package, PackageType, PackageState, db, PackageRelease +from app.models import Package, PackageType, PackageState, db, PackageRelease @bp.route("/packages///hub/") diff --git a/app/blueprints/packages/packages.py b/app/blueprints/packages/packages.py index e2c2911e..94195465 100644 --- a/app/blueprints/packages/packages.py +++ b/app/blueprints/packages/packages.py @@ -14,29 +14,37 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . +import datetime +import typing from urllib.parse import quote as urlescape from celery import uuid -from flask import render_template, make_response -from flask_login import login_required +from flask import render_template, make_response, request, redirect, flash, url_for, abort +from flask_babel import gettext, lazy_gettext +from flask_login import login_required, current_user from flask_wtf import FlaskForm from jinja2.utils import markupsafe -from sqlalchemy import func +from sqlalchemy import func, or_, and_ from sqlalchemy.orm import joinedload, subqueryload -from wtforms import * -from wtforms.validators import * +from wtforms import SelectField, StringField, TextAreaField, IntegerField, SubmitField, BooleanField +from wtforms.validators import InputRequired, Length, Regexp, DataRequired, Optional, URL, NumberRange, ValidationError from wtforms_sqlalchemy.fields import QuerySelectField, QuerySelectMultipleField from app.logic.LogicError import LogicError from app.logic.packages import do_edit_package -from app.models.packages import PackageProvides from app.querybuilder import QueryBuilder from app.rediscache import has_key, set_key from app.tasks.importtasks import importRepoScreenshot, checkZipRelease from app.tasks.webhooktasks import post_discord_webhook -from app.utils import * +from app.logic.game_support import GameSupportResolver + from . import bp, get_package_tabs -from ...logic.game_support import GameSupportResolver +from app.models import Package, Tag, db, User, Tags, PackageState, Permission, PackageType, MetaPackage, ForumTopic, \ + Dependency, Thread, UserRank, PackageReview, PackageDevState, ContentWarning, License, AuditSeverity, \ + PackageScreenshot, NotificationType, AuditLogEntry, PackageAlias, PackageProvides, PackageGameSupport, \ + PackageDailyStats +from app.utils import is_user_bot, get_int_or_abort, is_package_page, abs_url_for, addAuditLog, getPackageByInfo, \ + addNotification, get_system_user, rank_required, get_games_from_csv, get_daterange_options @bp.route("/packages/") @@ -174,14 +182,14 @@ def view(package): topic_error = "
".join(errors) - threads = Thread.query.filter_by(package_id=package.id, review_id=None) if not current_user.is_authenticated: threads = threads.filter_by(private=False) elif not current_user.rank.atLeast(UserRank.APPROVER) and not current_user == package.author: threads = threads.filter(or_(Thread.private == False, Thread.author == current_user)) - has_review = current_user.is_authenticated and PackageReview.query.filter_by(package=package, author=current_user).count() > 0 + has_review = current_user.is_authenticated and \ + PackageReview.query.filter_by(package=package, author=current_user).count() > 0 return render_template("packages/view.html", package=package, releases=releases, packages_uses=packages_uses, @@ -197,10 +205,11 @@ def shield(package, type): url = "https://img.shields.io/static/v1?label=ContentDB&message={}&color={}" \ .format(urlescape(package.title), urlescape("#375a7f")) elif type == "downloads": - api_url = abs_url_for("api.package", author=package.author.username, name=package.name) + api_url = abs_url_for("api.package_view", author=package.author.username, name=package.name) url = "https://img.shields.io/badge/dynamic/json?color={}&label=ContentDB&query=downloads&suffix=+downloads&url={}" \ .format(urlescape("#375a7f"), urlescape(api_url)) else: + from flask import abort abort(404) return redirect(url) @@ -213,7 +222,7 @@ def download(package): if release is None: if "application/zip" in request.accept_mimetypes and \ - not "text/html" in request.accept_mimetypes: + "text/html" not in request.accept_mimetypes: return "", 204 else: flash(gettext("No download available."), "danger") @@ -247,7 +256,7 @@ class PackageForm(FlaskForm): repo = StringField(lazy_gettext("VCS Repository URL"), [Optional(), URL()], filters = [lambda x: x or None]) website = StringField(lazy_gettext("Website URL"), [Optional(), URL()], filters = [lambda x: x or None]) issueTracker = StringField(lazy_gettext("Issue Tracker URL"), [Optional(), URL()], filters = [lambda x: x or None]) - forums = IntegerField(lazy_gettext("Forum Topic ID"), [Optional(), NumberRange(0,999999)]) + forums = IntegerField(lazy_gettext("Forum Topic ID"), [Optional(), NumberRange(0, 999999)]) video_url = StringField(lazy_gettext("Video URL"), [Optional(), URL()], filters=[lambda x: x or None]) donate_url = StringField(lazy_gettext("Donate URL"), [Optional(), URL()], filters=[lambda x: x or None]) @@ -721,7 +730,7 @@ def game_support(package): all_game_support = package.supported_games.all() all_game_support.sort(key=lambda x: -x.game.score) - supported_games_list: List[str] = [x.game.name for x in all_game_support if x.supports] + supported_games_list: typing.List[str] = [x.game.name for x in all_game_support if x.supports] if package.supports_all_games: supported_games_list.insert(0, "*") supported_games = ", ".join(supported_games_list) @@ -752,7 +761,7 @@ def statistics(package): @bp.route("/packages///stats.csv") @is_package_page def stats_csv(package): - stats: List[PackageDailyStats] = package.daily_stats.order_by(db.asc(PackageDailyStats.date)).all() + stats: typing.List[PackageDailyStats] = package.daily_stats.order_by(db.asc(PackageDailyStats.date)).all() columns = ["platform_minetest", "platform_other", "reason_new", "reason_dependency", "reason_update"] diff --git a/app/blueprints/packages/releases.py b/app/blueprints/packages/releases.py index 80a395b5..007f523a 100644 --- a/app/blueprints/packages/releases.py +++ b/app/blueprints/packages/releases.py @@ -14,19 +14,20 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . - -from flask import * -from flask_babel import gettext, lazy_gettext -from flask_login import login_required +from flask import render_template, request, redirect, flash, url_for, abort +from flask_babel import lazy_gettext, gettext +from flask_login import login_required, current_user from flask_wtf import FlaskForm -from wtforms import * +from wtforms import StringField, SubmitField, BooleanField, RadioField, FileField +from wtforms.validators import InputRequired, Length, Optional from wtforms_sqlalchemy.fields import QuerySelectField -from wtforms.validators import * from app.logic.releases import do_create_vcs_release, LogicError, do_create_zip_release +from app.models import Package, db, User, PackageState, Permission, UserRank, PackageDailyStats, MinetestRelease, \ + PackageRelease, PackageUpdateTrigger, PackageUpdateConfig from app.rediscache import has_key, set_key, make_download_key from app.tasks.importtasks import check_update_config -from app.utils import * +from app.utils import is_user_bot, is_package_page, nonEmptyOrNone from . import bp, get_package_tabs diff --git a/app/blueprints/packages/reviews.py b/app/blueprints/packages/reviews.py index dc6d18b5..1acbb7ad 100644 --- a/app/blueprints/packages/reviews.py +++ b/app/blueprints/packages/reviews.py @@ -13,22 +13,22 @@ # # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . + from collections import namedtuple +from flask import render_template, request, redirect, flash, url_for, abort from flask_babel import gettext, lazy_gettext - -from . import bp - -from flask import * from flask_login import current_user, login_required from flask_wtf import FlaskForm -from wtforms import * -from wtforms.validators import * +from wtforms import StringField, TextAreaField, SubmitField, RadioField +from wtforms.validators import InputRequired, Length + from app.models import db, PackageReview, Thread, ThreadReply, NotificationType, PackageReviewVote, Package, UserRank, \ Permission, AuditSeverity, PackageState +from app.tasks.webhooktasks import post_discord_webhook from app.utils import is_package_page, addNotification, get_int_or_abort, isYes, is_safe_url, rank_required, \ addAuditLog, has_blocked_domains -from app.tasks.webhooktasks import post_discord_webhook +from . import bp @bp.route("/reviews/") @@ -41,7 +41,7 @@ def list_reviews(): class ReviewForm(FlaskForm): - title = StringField(lazy_gettext("Title"), [InputRequired(), Length(3,100)]) + title = StringField(lazy_gettext("Title"), [InputRequired(), Length(3,100)]) comment = TextAreaField(lazy_gettext("Comment"), [InputRequired(), Length(10, 2000)]) rating = RadioField(lazy_gettext("Rating"), [InputRequired()], choices=[("5", lazy_gettext("Yes")), ("3", lazy_gettext("Neutral")), ("1", lazy_gettext("No"))]) @@ -248,5 +248,5 @@ def review_votes(package): user_biases_info.sort(key=lambda x: -abs(x.balance)) - return render_template("packages/review_votes.html", form=form, package=package, reviews=package.reviews, + return render_template("packages/review_votes.html", package=package, reviews=package.reviews, user_biases=user_biases_info) diff --git a/app/blueprints/packages/screenshots.py b/app/blueprints/packages/screenshots.py index 8c0d261b..d1c47785 100644 --- a/app/blueprints/packages/screenshots.py +++ b/app/blueprints/packages/screenshots.py @@ -14,19 +14,19 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . - -from flask import * -from flask_babel import gettext, lazy_gettext +from flask import render_template, request, redirect, flash, url_for, abort +from flask_babel import lazy_gettext, gettext +from flask_login import login_required, current_user from flask_wtf import FlaskForm -from flask_login import login_required -from wtforms import * +from wtforms import StringField, SubmitField, BooleanField, FileField +from wtforms.validators import InputRequired, Length, DataRequired, Optional from wtforms_sqlalchemy.fields import QuerySelectField -from wtforms.validators import * -from app.utils import * -from . import bp, get_package_tabs from app.logic.LogicError import LogicError from app.logic.screenshots import do_create_screenshot, do_order_screenshots +from . import bp, get_package_tabs +from app.models import Permission, db, PackageScreenshot +from app.utils import is_package_page class CreateScreenshotForm(FlaskForm): @@ -100,23 +100,23 @@ def edit_screenshot(package, id): if screenshot is None or screenshot.package != package: abort(404) - canEdit = package.check_perm(current_user, Permission.ADD_SCREENSHOTS) - canApprove = package.check_perm(current_user, Permission.APPROVE_SCREENSHOT) - if not (canEdit or canApprove): + can_edit = package.check_perm(current_user, Permission.ADD_SCREENSHOTS) + can_approve = package.check_perm(current_user, Permission.APPROVE_SCREENSHOT) + if not (can_edit or can_approve): return redirect(package.get_url("packages.screenshots")) # Initial form class from post data and default data form = EditScreenshotForm(obj=screenshot) if form.validate_on_submit(): - wasApproved = screenshot.approved + was_approved = screenshot.approved - if canEdit: + if can_edit: screenshot.title = form["title"].data or "Untitled" - if canApprove: + if can_approve: screenshot.approved = form["approved"].data else: - screenshot.approved = wasApproved + screenshot.approved = was_approved db.session.commit() return redirect(package.get_url("packages.screenshots")) diff --git a/app/blueprints/tasks/__init__.py b/app/blueprints/tasks/__init__.py index 0753a42e..358e7ed2 100644 --- a/app/blueprints/tasks/__init__.py +++ b/app/blueprints/tasks/__init__.py @@ -14,14 +14,14 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . - -from flask import * -from flask_login import login_required +from flask import Blueprint, jsonify, url_for, request, redirect, render_template +from flask_login import login_required, current_user from app import csrf +from app.models import UserRank from app.tasks import celery from app.tasks.importtasks import getMeta -from app.utils import * +from app.utils import shouldReturnJson bp = Blueprint("tasks", __name__) @@ -30,6 +30,7 @@ bp = Blueprint("tasks", __name__) @bp.route("/tasks/getmeta/new/", methods=["POST"]) @login_required def start_getmeta(): + from flask import request author = request.args.get("author") author = current_user.forums_username if author is None else author aresult = getMeta.delay(request.args.get("url"), author) diff --git a/app/blueprints/threads/__init__.py b/app/blueprints/threads/__init__.py index 502e860a..bc7b6220 100644 --- a/app/blueprints/threads/__init__.py +++ b/app/blueprints/threads/__init__.py @@ -13,7 +13,8 @@ # # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . -from flask import * + +from flask import Blueprint, request, render_template, abort, flash, redirect, url_for from flask_babel import gettext, lazy_gettext from app.markdown import get_user_mentions, render_markdown @@ -22,11 +23,12 @@ from app.tasks.webhooktasks import post_discord_webhook bp = Blueprint("threads", __name__) from flask_login import current_user, login_required -from app.models import * +from app.models import Package, db, User, Permission, Thread, UserRank, AuditSeverity, \ + NotificationType, ThreadReply from app.utils import addNotification, isYes, addAuditLog, get_system_user, rank_required, has_blocked_domains from flask_wtf import FlaskForm -from wtforms import * -from wtforms.validators import * +from wtforms import StringField, TextAreaField, SubmitField, BooleanField +from wtforms.validators import InputRequired, Length from app.utils import get_int_or_abort @@ -58,8 +60,8 @@ def list_all(): @bp.route("/threads//subscribe/", methods=["POST"]) @login_required -def subscribe(id): - thread = Thread.query.get(id) +def subscribe(id_): + thread = Thread.query.get(id_) if thread is None or not thread.check_perm(current_user, Permission.SEE_THREAD): abort(404) @@ -75,8 +77,8 @@ def subscribe(id): @bp.route("/threads//unsubscribe/", methods=["POST"]) @login_required -def unsubscribe(id): - thread = Thread.query.get(id) +def unsubscribe(id_): + thread = Thread.query.get(id_) if thread is None or not thread.check_perm(current_user, Permission.SEE_THREAD): abort(404) @@ -92,8 +94,8 @@ def unsubscribe(id): @bp.route("/threads//set-lock/", methods=["POST"]) @login_required -def set_lock(id): - thread = Thread.query.get(id) +def set_lock(id_): + thread = Thread.query.get(id_) if thread is None or not thread.check_perm(current_user, Permission.LOCK_THREAD): abort(404) @@ -118,8 +120,8 @@ def set_lock(id): @bp.route("/threads//delete/", methods=["GET", "POST"]) @login_required -def delete_thread(id): - thread = Thread.query.get(id) +def delete_thread(id_): + thread = Thread.query.get(id_) if thread is None or not thread.check_perm(current_user, Permission.DELETE_THREAD): abort(404) @@ -141,8 +143,8 @@ def delete_thread(id): @bp.route("/threads//delete-reply/", methods=["GET", "POST"]) @login_required -def delete_reply(id): - thread = Thread.query.get(id) +def delete_reply(id_): + thread = Thread.query.get(id_) if thread is None: abort(404) @@ -180,8 +182,8 @@ class CommentForm(FlaskForm): @bp.route("/threads//edit/", methods=["GET", "POST"]) @login_required -def edit_reply(id): - thread = Thread.query.get(id) +def edit_reply(id_): + thread = Thread.query.get(id_) if thread is None: abort(404) @@ -217,8 +219,8 @@ def edit_reply(id): @bp.route("/threads//", methods=["GET", "POST"]) -def view(id): - thread: Thread = Thread.query.get(id) +def view(id_): + thread: Thread = Thread.query.get(id_) if thread is None or not thread.check_perm(current_user, Permission.SEE_THREAD): abort(404) @@ -377,7 +379,6 @@ def new(): return redirect(thread.get_view_url()) - return render_template("threads/new.html", form=form, allow_private_change=allow_private_change, package=package) diff --git a/app/blueprints/users/account.py b/app/blueprints/users/account.py index 566347ac..ad198e4a 100644 --- a/app/blueprints/users/account.py +++ b/app/blueprints/users/account.py @@ -14,21 +14,22 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . +import datetime - -from flask import * -from flask_babel import gettext, get_locale +from flask import redirect, abort, render_template, flash, request, url_for +from flask_babel import gettext, get_locale, lazy_gettext from flask_login import current_user, login_required, logout_user, login_user from flask_wtf import FlaskForm from sqlalchemy import or_ -from wtforms import * -from wtforms.validators import * +from wtforms import StringField, SubmitField, BooleanField, PasswordField, validators +from wtforms.validators import InputRequired, Length, Regexp, DataRequired, Optional, Email, EqualTo -from app.models import * from app.tasks.emails import send_verify_email, send_anon_email, send_unsubscribe_verify, send_user_email from app.utils import randomString, make_flask_login_password, is_safe_url, check_password_hash, addAuditLog, \ nonEmptyOrNone, post_login, is_username_valid from . import bp +from app.models import User, AuditSeverity, db, UserRank, PackageAlias, EmailSubscription, UserNotificationPreferences, \ + UserEmailVerification class LoginForm(FlaskForm): @@ -45,7 +46,6 @@ def handle_login(form): else: flash(err, "danger") - username = form.username.data.strip() user = User.query.filter(or_(User.username == username, User.email == username)).first() if user is None: @@ -87,7 +87,6 @@ def login(): if request.method == "GET": form.remember_me.data = True - return render_template("users/login.html", form=form) @@ -187,6 +186,7 @@ class ForgotPasswordForm(FlaskForm): email = StringField(lazy_gettext("Email"), [InputRequired(), Email()]) submit = SubmitField(lazy_gettext("Reset Password")) + @bp.route("/user/forgot-password/", methods=["GET", "POST"]) def forgot_password(): form = ForgotPasswordForm(request.form) @@ -222,9 +222,10 @@ class SetPasswordForm(FlaskForm): email = StringField(lazy_gettext("Email"), [Optional(), Email()]) password = PasswordField(lazy_gettext("New password"), [InputRequired(), Length(8, 100)]) password2 = PasswordField(lazy_gettext("Verify password"), [InputRequired(), Length(8, 100), - validators.EqualTo('password', message=lazy_gettext('Passwords must match'))]) + EqualTo('password', message=lazy_gettext('Passwords must match'))]) submit = SubmitField(lazy_gettext("Save")) + class ChangePasswordForm(FlaskForm): old_password = PasswordField(lazy_gettext("Old password"), [InputRequired(), Length(8, 100)]) password = PasswordField(lazy_gettext("New password"), [InputRequired(), Length(8, 100)]) @@ -245,8 +246,8 @@ def handle_set_password(form): current_user.password = make_flask_login_password(form.password.data) if hasattr(form, "email"): - newEmail = nonEmptyOrNone(form.email.data) - if newEmail and newEmail != current_user.email: + new_email = nonEmptyOrNone(form.email.data) + if new_email and new_email != current_user.email: if EmailSubscription.query.filter_by(email=form.email.data, blacklisted=True).count() > 0: flash(gettext(u"That email address has been unsubscribed/blacklisted, and cannot be used"), "danger") return @@ -262,7 +263,7 @@ def handle_set_password(form): ver = UserEmailVerification() ver.user = current_user ver.token = token - ver.email = newEmail + ver.email = new_email db.session.add(ver) db.session.commit() diff --git a/app/blueprints/users/claim.py b/app/blueprints/users/claim.py index 6949a282..52ef4adf 100644 --- a/app/blueprints/users/claim.py +++ b/app/blueprints/users/claim.py @@ -13,6 +13,7 @@ # # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . + from flask_babel import gettext from . import bp @@ -58,7 +59,7 @@ def claim_forums(): session["forum_token"] = token if request.method == "POST": - ctype = request.form.get("claim_type") + ctype = request.form.get("claim_type") username = request.form.get("username") if not is_username_valid(username): diff --git a/app/blueprints/users/profile.py b/app/blueprints/users/profile.py index dce08a72..b4cad6ca 100644 --- a/app/blueprints/users/profile.py +++ b/app/blueprints/users/profile.py @@ -15,16 +15,16 @@ # along with this program. If not, see . import math -from typing import Optional +from typing import Optional, Tuple, List -from flask import * +from flask import redirect, url_for, abort, render_template, request from flask_babel import gettext from flask_login import current_user, login_required -from sqlalchemy import func +from sqlalchemy import func, text -from app.models import * +from app.models import User, db, Package, PackageReview, PackageState, PackageType +from app.utils import get_daterange_options from . import bp -from ...utils import get_daterange_options @bp.route("/users/", methods=["GET"]) diff --git a/app/blueprints/users/settings.py b/app/blueprints/users/settings.py index aec179eb..48124bea 100644 --- a/app/blueprints/users/settings.py +++ b/app/blueprints/users/settings.py @@ -1,12 +1,29 @@ -from flask import * -from flask_babel import gettext, get_locale +# ContentDB +# Copyright (C) 2023 rubenwardy +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +from flask import redirect, abort, render_template, request, flash, url_for +from flask_babel import gettext, get_locale, lazy_gettext from flask_login import current_user, login_required, logout_user from flask_wtf import FlaskForm from sqlalchemy import or_ -from wtforms import * -from wtforms.validators import * +from wtforms import StringField, SubmitField, BooleanField, SelectField +from wtforms.validators import Length, Optional, Email, URL -from app.models import * +from app.models import User, AuditSeverity, db, UserRank, PackageAlias, EmailSubscription, UserNotificationPreferences, \ + UserEmailVerification, Permission, NotificationType, UserBan from app.tasks.emails import send_verify_email from app.utils import nonEmptyOrNone, addAuditLog, randomString, rank_required, has_blocked_domains from . import bp @@ -309,9 +326,9 @@ def modtools(username): user.github_username = nonEmptyOrNone(form.github_username.data) if user.check_perm(current_user, Permission.CHANGE_RANK): - newRank = form["rank"].data - if current_user.rank.atLeast(newRank): - if newRank != user.rank: + new_rank = form["rank"].data + if current_user.rank.atLeast(new_rank): + if new_rank != user.rank: user.rank = form["rank"].data msg = "Set rank of {} to {}".format(user.display_name, user.rank.get_title()) addAuditLog(AuditSeverity.MODERATION, current_user, msg, diff --git a/app/blueprints/zipgrep/__init__.py b/app/blueprints/zipgrep/__init__.py index 64856dc1..b5d74b60 100644 --- a/app/blueprints/zipgrep/__init__.py +++ b/app/blueprints/zipgrep/__init__.py @@ -16,7 +16,8 @@ from celery import uuid -from flask import Blueprint, render_template, redirect, request, abort +from flask import Blueprint, render_template, redirect, request, abort, url_for +from flask_babel import lazy_gettext from flask_wtf import FlaskForm from wtforms import StringField, BooleanField, SubmitField from wtforms.validators import InputRequired, Length @@ -26,7 +27,7 @@ from app.utils import rank_required bp = Blueprint("zipgrep", __name__) -from app.models import * +from app.models import UserRank, Package from app.tasks.zipgrep import search_in_releases @@ -51,14 +52,14 @@ def zipgrep_search(): @bp.route("/zipgrep//") -def view_results(id): - result = celery.AsyncResult(id) +def view_results(id_): + result = celery.AsyncResult(id_) if result.status == "PENDING": abort(404) if result.status != "SUCCESS" or isinstance(result.result, Exception): - result_url = url_for("zipgrep.view_results", id=id) - return redirect(url_for("tasks.check", id=id, r=result_url)) + result_url = url_for("zipgrep.view_results", id=id_) + return redirect(url_for("tasks.check", id=id_, r=result_url)) matches = result.result["matches"] for match in matches: diff --git a/app/default_data.py b/app/default_data.py index c5f12abe..63dd8770 100644 --- a/app/default_data.py +++ b/app/default_data.py @@ -1,4 +1,23 @@ -from .models import * +# ContentDB +# Copyright (C) rubenwardy +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +import datetime + +from .models import User, UserRank, MinetestRelease, Tag, License, Notification, NotificationType, Package, \ + PackageState, PackageType, PackageRelease, MetaPackage, Dependency from .utils import make_flask_login_password @@ -68,7 +87,6 @@ def populate_test_data(session): jeija.forums_username = "Jeija" session.add(jeija) - mod = Package() mod.state = PackageState.APPROVED mod.name = "alpha" diff --git a/app/flatpages/about.md b/app/flatpages/about.md index a094e710..48886538 100644 --- a/app/flatpages/about.md +++ b/app/flatpages/about.md @@ -30,3 +30,10 @@ and texture packs for Minetest**. You should read [the official Minetest Modding Book](https://rubenwardy.com/minetest_modding_book/) for a guide to making mods and games using Minetest. + +## How can I support / donate to ContentDB? + +You can donate to rubenwardy to cover ContentDB's costs and support future +development. + +Donate diff --git a/app/flatpages/help/api.md b/app/flatpages/help/api.md index 6c1243d7..c06f338f 100644 --- a/app/flatpages/help/api.md +++ b/app/flatpages/help/api.md @@ -334,7 +334,7 @@ curl -X POST https://content.minetest.net/api/packages/username/name/screenshots * `author`: filter by review author username * `rating`: 1 for negative, 3 for neutral, 5 for positive * `is_positive`: true or false. Default: null - * `q`: filter by title (case insensitive, no fulltext search) + * `q`: filter by title (case-insensitive, no fulltext search) Example: diff --git a/app/logic/graphs.py b/app/logic/graphs.py index d5bcacba..b464568a 100644 --- a/app/logic/graphs.py +++ b/app/logic/graphs.py @@ -1,3 +1,19 @@ +# ContentDB +# Copyright (C) rubenwardy +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + import datetime from datetime import timedelta from typing import Optional @@ -15,7 +31,7 @@ keys = ["platform_minetest", "platform_other", "reason_new", "reason_dependency", "reason_update"] -def _flatten_data(stats): +def flatten_data(stats): start_date = stats[0].date end_date = stats[-1].date result = { @@ -52,7 +68,7 @@ def get_package_stats(package: Package, start_date: Optional[datetime.date], end if len(stats) == 0: return None - return _flatten_data(stats) + return flatten_data(stats) def get_package_stats_for_user(user: User, start_date: Optional[datetime.date], end_date: Optional[datetime.date]): @@ -76,7 +92,7 @@ def get_package_stats_for_user(user: User, start_date: Optional[datetime.date], if len(stats) == 0: return None - results = _flatten_data(stats) + results = flatten_data(stats) results["package_downloads"] = get_package_overview_for_user(user, stats[0].date, stats[-1].date) return results diff --git a/app/logic/package_validator.py b/app/logic/package_validator.py index 10d07905..5f1be3e0 100644 --- a/app/logic/package_validator.py +++ b/app/logic/package_validator.py @@ -1,3 +1,19 @@ +# ContentDB +# Copyright (C) rubenwardy +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + from collections import namedtuple from typing import List @@ -15,11 +31,11 @@ def validate_package_for_approval(package: Package) -> List[ValidationError]: normalised_name = package.getNormalisedName() if package.type != PackageType.MOD and Package.query.filter( and_(Package.state == PackageState.APPROVED, - or_(Package.name == normalised_name, - Package.name == normalised_name + "_game"))).count() > 0: + or_(Package.name == normalised_name, + Package.name == normalised_name + "_game"))).count() > 0: retval.append(("danger", lazy_gettext("A package already exists with this name. Please see Policy and Guidance 3"))) - if package.releases.filter(PackageRelease.task_id==None).count() == 0: + if package.releases.filter(PackageRelease.task_id == None).count() == 0: retval.append(("danger", lazy_gettext("A release is required before this package can be approved."))) # Don't bother validating any more until we have a release return retval diff --git a/app/logic/releases.py b/app/logic/releases.py index 52a4081f..8cc07681 100644 --- a/app/logic/releases.py +++ b/app/logic/releases.py @@ -14,8 +14,8 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . - -import datetime, re +import datetime +import re from celery import uuid from flask_babel import lazy_gettext diff --git a/app/logic/screenshots.py b/app/logic/screenshots.py index 869255c6..1f7171e6 100644 --- a/app/logic/screenshots.py +++ b/app/logic/screenshots.py @@ -1,3 +1,19 @@ +# ContentDB +# Copyright (C) rubenwardy +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + import datetime, json from flask_babel import lazy_gettext diff --git a/app/logic/uploads.py b/app/logic/uploads.py index 0795e838..a4849e22 100644 --- a/app/logic/uploads.py +++ b/app/logic/uploads.py @@ -14,44 +14,47 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . - import imghdr import os from flask_babel import lazy_gettext +from app import app from app.logic.LogicError import LogicError -from app.models import * from app.utils import randomString def get_extension(filename): return filename.rsplit(".", 1)[1].lower() if "." in filename else None + ALLOWED_IMAGES = {"jpeg", "png"} -def isAllowedImage(data): + + +def is_allowed_image(data): return imghdr.what(None, data) in ALLOWED_IMAGES -def upload_file(file, fileType, fileTypeDesc): + +def upload_file(file, file_type, file_type_desc): if not file or file is None or file.filename == "": raise LogicError(400, "Expected file") assert os.path.isdir(app.config["UPLOAD_DIR"]), "UPLOAD_DIR must exist" - isImage = False - if fileType == "image": - allowedExtensions = ["jpg", "jpeg", "png"] - isImage = True - elif fileType == "zip": - allowedExtensions = ["zip"] + is_image = False + if file_type == "image": + allowed_extensions = ["jpg", "jpeg", "png"] + is_image = True + elif file_type == "zip": + allowed_extensions = ["zip"] else: raise Exception("Invalid fileType") ext = get_extension(file.filename) - if ext is None or not ext in allowedExtensions: - raise LogicError(400, lazy_gettext("Please upload %(file_desc)s", file_desc=fileTypeDesc)) + if ext is None or ext not in allowed_extensions: + raise LogicError(400, lazy_gettext("Please upload %(file_desc)s", file_desc=file_type_desc)) - if isImage and not isAllowedImage(file.stream.read()): + if is_image and not is_allowed_image(file.stream.read()): raise LogicError(400, lazy_gettext("Uploaded image isn't actually an image")) file.stream.seek(0) diff --git a/app/maillogger.py b/app/maillogger.py index 52fb6214..687a5081 100644 --- a/app/maillogger.py +++ b/app/maillogger.py @@ -1,3 +1,19 @@ +# ContentDB +# Copyright (C) rubenwardy +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + import logging from app.tasks.emails import send_user_email @@ -9,6 +25,7 @@ def _has_newline(line): return True return False + def _is_bad_subject(subject): """Copied from: flask_mail.py class Message def has_bad_headers""" if _has_newline(subject): @@ -32,9 +49,11 @@ class FlaskMailSubjectFormatter(logging.Formatter): s = self.formatMessage(record) return s + class FlaskMailTextFormatter(logging.Formatter): pass + class FlaskMailHTMLFormatter(logging.Formatter): def formatException(self, exc_info): formatted_exception = logging.Handler.formatException(self, exc_info) diff --git a/app/markdown.py b/app/markdown.py index 68152068..a0d565e3 100644 --- a/app/markdown.py +++ b/app/markdown.py @@ -1,3 +1,19 @@ +# ContentDB +# Copyright (C) rubenwardy +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + from functools import partial import bleach diff --git a/app/models/__init__.py b/app/models/__init__.py index ac8aa44c..dbe10347 100644 --- a/app/models/__init__.py +++ b/app/models/__init__.py @@ -47,7 +47,7 @@ class APIToken(db.Model): package_id = db.Column(db.Integer, db.ForeignKey("package.id"), nullable=True) package = db.relationship("Package", foreign_keys=[package_id], back_populates="tokens") - def canOperateOnPackage(self, package): + def can_operate_on_package(self, package): if self.package and self.package != package: return False @@ -146,7 +146,7 @@ class ForumTopic(db.Model): created_at = db.Column(db.DateTime, nullable=False, default=datetime.datetime.utcnow) - def getRepoURL(self): + def get_repo_url(self): if self.link is None: return None diff --git a/app/models/packages.py b/app/models/packages.py index b6bfc686..d73da5de 100644 --- a/app/models/packages.py +++ b/app/models/packages.py @@ -27,7 +27,7 @@ from sqlalchemy.dialects.postgresql import insert from . import db from .users import Permission, UserRank, User -from .. import app +from app import app class PackageQuery(BaseQuery, SearchQueryMixin): diff --git a/app/models/threads.py b/app/models/threads.py index 154c4d8c..54146018 100644 --- a/app/models/threads.py +++ b/app/models/threads.py @@ -68,7 +68,7 @@ class Thread(db.Model): def get_view_url(self, absolute=False): if absolute: - from ..utils import abs_url_for + from app.utils import abs_url_for return abs_url_for("threads.view", id=self.id) else: return url_for("threads.view", id=self.id, _external=False) diff --git a/app/models/users.py b/app/models/users.py index ce22d2f0..1dc7670a 100644 --- a/app/models/users.py +++ b/app/models/users.py @@ -14,7 +14,6 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . - import datetime import enum @@ -351,7 +350,7 @@ class EmailSubscription(db.Model): @property def url(self): - from ..utils import abs_url_for + from app.utils import abs_url_for return abs_url_for('users.unsubscribe', token=self.token) diff --git a/app/querybuilder.py b/app/querybuilder.py index 59d5bc15..be63b5e1 100644 --- a/app/querybuilder.py +++ b/app/querybuilder.py @@ -1,3 +1,19 @@ +# ContentDB +# Copyright (C) rubenwardy +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + from flask import abort, current_app from flask_babel import lazy_gettext from sqlalchemy import or_ diff --git a/app/rediscache.py b/app/rediscache.py index 01aaf5b0..869abb04 100644 --- a/app/rediscache.py +++ b/app/rediscache.py @@ -1,3 +1,19 @@ +# ContentDB +# Copyright (C) rubenwardy +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + from . import r # This file acts as a facade between the releases code and redis, diff --git a/app/tasks/__init__.py b/app/tasks/__init__.py index 5e419430..f93baae8 100644 --- a/app/tasks/__init__.py +++ b/app/tasks/__init__.py @@ -25,6 +25,7 @@ from app import app class TaskError(Exception): def __init__(self, value): self.value = value + def __str__(self): return repr("TaskError: " + self.value) diff --git a/app/tasks/forumtasks.py b/app/tasks/forumtasks.py index 1aea4310..dad2617d 100644 --- a/app/tasks/forumtasks.py +++ b/app/tasks/forumtasks.py @@ -14,14 +14,16 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . +import json +import re +import sys +import urllib.request +from urllib.parse import urljoin -import json, re, sys -from app.models import * +from app.models import User, db, PackageType, ForumTopic from app.tasks import celery from app.utils import is_username_valid from app.utils.phpbbparser import getProfile, getTopicsFromForum -import urllib.request -from urllib.parse import urljoin from .usertasks import set_profile_picture_from_url @@ -84,6 +86,8 @@ def checkAllForumAccounts(): regex_tag = re.compile(r"\[([a-z0-9_]+)\]") BANNED_NAMES = ["mod", "game", "old", "outdated", "wip", "api", "beta", "alpha", "git"] + + def getNameFromTaglist(taglist): for tag in reversed(regex_tag.findall(taglist)): if len(tag) < 30 and not tag in BANNED_NAMES and \ @@ -92,7 +96,10 @@ def getNameFromTaglist(taglist): return None + regex_title = re.compile(r"^((?:\[[^\]]+\] *)*)([^\[]+) *((?:\[[^\]]+\] *)*)[^\[]*$") + + def parseTitle(title): m = regex_title.match(title) if m is None: diff --git a/app/tasks/importtasks.py b/app/tasks/importtasks.py index f3a8e663..4199f293 100644 --- a/app/tasks/importtasks.py +++ b/app/tasks/importtasks.py @@ -13,27 +13,31 @@ # # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . -from json import JSONDecodeError -import gitdb +import datetime import json import os import shutil +from json import JSONDecodeError from zipfile import ZipFile +import gitdb +from flask import url_for from git import GitCommandError from git_archive_all import GitArchiver from kombu import uuid -from app.models import * +from app.models import AuditSeverity, db, NotificationType, PackageRelease, MetaPackage, Dependency, PackageType, \ + MinetestRelease, Package, PackageState, PackageScreenshot, PackageUpdateTrigger, PackageUpdateConfig from app.tasks import celery, TaskError from app.utils import randomString, post_bot_message, addSystemNotification, addSystemAuditLog, get_games_from_csv from app.utils.git import clone_repo, get_latest_tag, get_latest_commit, get_temp_dir from .minetestcheck import build_tree, MinetestCheckError, ContentType -from ..logic.LogicError import LogicError -from ..logic.game_support import GameSupportResolver -from ..logic.packages import do_edit_package, ALIASES -from ..utils.image import get_image_size +from app import app +from app.logic.LogicError import LogicError +from app.logic.game_support import GameSupportResolver +from app.logic.packages import do_edit_package, ALIASES +from app.utils.image import get_image_size @celery.task() @@ -51,7 +55,7 @@ def getMeta(urlstr, author): result["forums"] = result.get("forumId") - readme_path = tree.getReadMePath() + readme_path = tree.get_readme_path() if readme_path: with open(readme_path, "r") as f: result["long_description"] = f.read() @@ -96,11 +100,11 @@ def postReleaseCheckUpdate(self, release: PackageRelease, path): def getMetaPackages(names): return [ MetaPackage.GetOrCreate(x, cache) for x in names ] - provides = tree.getModNames() + provides = tree.get_mod_names() package = release.package package.provides.clear() - package.provides.extend(getMetaPackages(tree.getModNames())) + package.provides.extend(getMetaPackages(tree.get_mod_names())) # Delete all mod name dependencies package.dependencies.filter(Dependency.meta_package != None).delete() diff --git a/app/tasks/minetestcheck/__init__.py b/app/tasks/minetestcheck/__init__.py index 5d8e4ddc..97557c48 100644 --- a/app/tasks/minetestcheck/__init__.py +++ b/app/tasks/minetestcheck/__init__.py @@ -1,11 +1,30 @@ +# ContentDB +# Copyright (C) 2018-23 rubenwardy +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + from enum import Enum + class MinetestCheckError(Exception): def __init__(self, value): self.value = value + def __str__(self): return repr("Error validating package: " + self.value) + class ContentType(Enum): UNKNOWN = "unknown" MOD = "mod" @@ -13,7 +32,7 @@ class ContentType(Enum): GAME = "game" TXP = "texture pack" - def isModLike(self): + def is_mod_like(self): return self == ContentType.MOD or self == ContentType.MODPACK def validate_same(self, other): @@ -23,7 +42,7 @@ class ContentType(Enum): assert other if self == ContentType.MOD: - if not other.isModLike(): + if not other.is_mod_like(): raise MinetestCheckError("Expected a mod or modpack, found " + other.value) elif self == ContentType.TXP: @@ -36,6 +55,7 @@ class ContentType(Enum): from .tree import PackageTreeNode, get_base_dir + def build_tree(path, expected_type=None, author=None, repo=None, name=None): path = get_base_dir(path) diff --git a/app/tasks/minetestcheck/config.py b/app/tasks/minetestcheck/config.py index b5d0d351..eb670c5a 100644 --- a/app/tasks/minetestcheck/config.py +++ b/app/tasks/minetestcheck/config.py @@ -1,3 +1,20 @@ +# ContentDB +# Copyright (C) Lars Mueller +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + + def parse_conf(string): retval = {} lines = string.splitlines() diff --git a/app/tasks/minetestcheck/tree.py b/app/tasks/minetestcheck/tree.py index c16f23f1..4d7e3a01 100644 --- a/app/tasks/minetestcheck/tree.py +++ b/app/tasks/minetestcheck/tree.py @@ -1,9 +1,29 @@ -import os, re +# ContentDB +# Copyright (C) 2018-21 rubenwardy +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + + +import os +import re + from . import MinetestCheckError, ContentType from .config import parse_conf basenamePattern = re.compile("^([a-z0-9_]+)$") + def get_base_dir(path): if not os.path.isdir(path): raise IOError("Expected dir") @@ -39,8 +59,8 @@ def get_csv_line(line): class PackageTreeNode: - def __init__(self, baseDir, relative, author=None, repo=None, name=None): - self.baseDir = baseDir + def __init__(self, base_dir, relative, author=None, repo=None, name=None): + self.baseDir = base_dir self.relative = relative self.author = author self.name = name @@ -49,11 +69,11 @@ class PackageTreeNode: self.children = [] # Detect type - self.type = detect_type(baseDir) + self.type = detect_type(base_dir) self.read_meta() if self.type == ContentType.GAME: - if not os.path.isdir(baseDir + "/mods"): + if not os.path.isdir(base_dir + "/mods"): raise MinetestCheckError("Game at {} does not have a mods/ folder".format(self.relative)) self.add_children_from_mod_dir("mods") elif self.type == ContentType.MOD: @@ -69,13 +89,13 @@ class PackageTreeNode: if lowercase != dir and lowercase in dirs: raise MinetestCheckError(f"Incorrect case, {dir} should be {lowercase} at {self.relative}{dir}") - def getReadMePath(self): + def get_readme_path(self): for filename in os.listdir(self.baseDir): path = os.path.join(self.baseDir, filename) if os.path.isfile(path) and filename.lower().startswith("readme."): return path - def getMetaFileName(self): + def get_meta_file_name(self): if self.type == ContentType.GAME: return "game.conf" elif self.type == ContentType.MOD: @@ -91,13 +111,13 @@ class PackageTreeNode: result = {} # Read .conf file - meta_file_name = self.getMetaFileName() + meta_file_name = self.get_meta_file_name() if meta_file_name is not None: meta_file_rel = self.relative + meta_file_name meta_file_path = self.baseDir + "/" + meta_file_name try: - with open(meta_file_path or "", "r") as myfile: - conf = parse_conf(myfile.read()) + with open(meta_file_path or "", "r") as f: + conf = parse_conf(f.read()) for key, value in conf.items(): result[key] = value except SyntaxError as e: @@ -108,12 +128,11 @@ class PackageTreeNode: if "release" in result: raise MinetestCheckError("{} should not contain 'release' key, as this is for use by ContentDB only.".format(meta_file_rel)) - # description.txt - if not "description" in result: + if "description" not in result: try: - with open(self.baseDir + "/description.txt", "r") as myfile: - result["description"] = myfile.read() + with open(self.baseDir + "/description.txt", "r") as f: + result["description"] = f.read() except IOError: pass @@ -123,10 +142,10 @@ class PackageTreeNode: result["optional_depends"] = get_csv_line(result.get("optional_depends")) elif os.path.isfile(self.baseDir + "/depends.txt"): - pattern = re.compile("^([a-z0-9_]+)\??$") + pattern = re.compile(r"^([a-z0-9_]+)\??$") - with open(self.baseDir + "/depends.txt", "r") as myfile: - contents = myfile.read() + with open(self.baseDir + "/depends.txt", "r") as f: + contents = f.read() soft = [] hard = [] for line in contents.split("\n"): @@ -144,8 +163,7 @@ class PackageTreeNode: result["depends"] = [] result["optional_depends"] = [] - - def checkDependencies(deps): + def check_dependencies(deps): for dep in deps: if not basenamePattern.match(dep): if " " in dep: @@ -157,8 +175,8 @@ class PackageTreeNode: .format(dep, self.relative)) # Check dependencies - checkDependencies(result["depends"]) - checkDependencies(result["optional_depends"]) + check_dependencies(result["depends"]) + check_dependencies(result["optional_depends"]) # Fix games using "name" as "title" if self.type == ContentType.GAME and "name" in result: @@ -193,7 +211,7 @@ class PackageTreeNode: path = os.path.join(dir, entry) if not entry.startswith('.') and os.path.isdir(path): child = PackageTreeNode(path, relative + entry + "/", name=entry) - if not child.type.isModLike(): + if not child.type.is_mod_like(): raise MinetestCheckError("Expecting mod or modpack, found {} at {} inside {}" \ .format(child.type.value, child.relative, self.type.value)) @@ -202,7 +220,7 @@ class PackageTreeNode: self.children.append(child) - def getModNames(self): + def get_mod_names(self): return self.fold("name", type_=ContentType.MOD) # attr: Attribute name diff --git a/app/tasks/webhooktasks.py b/app/tasks/webhooktasks.py index a19a97d0..52252c08 100644 --- a/app/tasks/webhooktasks.py +++ b/app/tasks/webhooktasks.py @@ -13,7 +13,7 @@ # # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . -import sys + from typing import Optional import requests @@ -22,6 +22,7 @@ from app import app from app.models import User from app.tasks import celery + @celery.task() def post_discord_webhook(username: Optional[str], content: str, is_queue: bool, title: Optional[str] = None, description: Optional[str] = None, thumbnail: Optional[str] = None): discord_url = app.config.get("DISCORD_WEBHOOK_QUEUE" if is_queue else "DISCORD_WEBHOOK_FEED") diff --git a/app/tasks/zipgrep.py b/app/tasks/zipgrep.py index be5fb1ec..f4700f39 100644 --- a/app/tasks/zipgrep.py +++ b/app/tasks/zipgrep.py @@ -14,7 +14,6 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . - import subprocess from subprocess import Popen, PIPE from typing import Optional diff --git a/app/template_filters.py b/app/template_filters.py index c1b13065..87a38de0 100644 --- a/app/template_filters.py +++ b/app/template_filters.py @@ -1,3 +1,19 @@ +# ContentDB +# Copyright (C) rubenwardy +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + from datetime import datetime as dt from urllib.parse import urlparse diff --git a/app/tests/integ/test_api.py b/app/tests/integ/test_api.py index a759db26..aa4c1535 100644 --- a/app/tests/integ/test_api.py +++ b/app/tests/integ/test_api.py @@ -1,3 +1,20 @@ +# ContentDB +# Copyright (C) rubenwardy +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + + from app.default_data import populate_test_data from app.models import db, Package, PackageState from .utils import parse_json, validate_package_list @@ -14,7 +31,6 @@ def test_packages_empty(client): def test_packages_with_contents(client): """Start with a test database.""" - populate_test_data(db.session) db.session.commit() diff --git a/app/tests/integ/test_homepage.py b/app/tests/integ/test_homepage.py index fd0e8116..c80bb2ae 100644 --- a/app/tests/integ/test_homepage.py +++ b/app/tests/integ/test_homepage.py @@ -1,3 +1,19 @@ +# ContentDB +# Copyright (C) rubenwardy +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + from app.default_data import populate_test_data from app.models import db from .utils import client # noqa diff --git a/app/tests/integ/test_user.py b/app/tests/integ/test_user.py index 334d0d60..0e8c3a93 100644 --- a/app/tests/integ/test_user.py +++ b/app/tests/integ/test_user.py @@ -1,3 +1,19 @@ +# ContentDB +# Copyright (C) rubenwardy +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + from flask import url_for from app.models import User, UserEmailVerification diff --git a/app/tests/integ/utils.py b/app/tests/integ/utils.py index 8b80ad5d..adcc6536 100644 --- a/app/tests/integ/utils.py +++ b/app/tests/integ/utils.py @@ -1,3 +1,19 @@ +# ContentDB +# Copyright (C) rubenwardy +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + import pytest, json from sqlalchemy import text @@ -19,18 +35,23 @@ def recreate_db(): populate(db.session) db.session.commit() + def parse_json(b): return json.loads(b.decode("utf8")) + def is_type(t, v): return v and isinstance(v, t) + def is_optional(t, v): return not v or isinstance(v, t) + def is_str(v): return is_type(str, v) + def is_int(v): return is_type(int, v) diff --git a/app/tests/unit/test_git.py b/app/tests/unit/test_git.py index 3a970bbf..714160c0 100644 --- a/app/tests/unit/test_git.py +++ b/app/tests/unit/test_git.py @@ -1,3 +1,19 @@ +# ContentDB +# Copyright (C) rubenwardy +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + import os from app.utils.git import get_latest_tag, get_latest_commit, clone_repo diff --git a/app/tests/unit/test_logic_graphs.py b/app/tests/unit/test_logic_graphs.py index 9ed1cbf1..c7a76227 100644 --- a/app/tests/unit/test_logic_graphs.py +++ b/app/tests/unit/test_logic_graphs.py @@ -1,6 +1,22 @@ +# ContentDB +# Copyright (C) rubenwardy +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + import datetime -from app.logic.graphs import _flatten_data +from app.logic.graphs import flatten_data class DailyStat: @@ -21,7 +37,7 @@ class DailyStat: def test_flatten_data(): - res = _flatten_data([ + res = flatten_data([ DailyStat("2022-03-28", 3), DailyStat("2022-03-29", 10), DailyStat("2022-04-01", 5), diff --git a/app/tests/unit/test_minetest_hypertext.py b/app/tests/unit/test_minetest_hypertext.py index 963ff589..4d6df866 100644 --- a/app/tests/unit/test_minetest_hypertext.py +++ b/app/tests/unit/test_minetest_hypertext.py @@ -1,3 +1,19 @@ +# ContentDB +# Copyright (C) rubenwardy +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + from app.utils.minetest_hypertext import html_to_minetest diff --git a/app/tests/unit/test_url.py b/app/tests/unit/test_url.py index 1510b073..11b3fc5a 100644 --- a/app/tests/unit/test_url.py +++ b/app/tests/unit/test_url.py @@ -1,3 +1,19 @@ +# ContentDB +# Copyright (C) rubenwardy +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + from app.utils.url import clean_youtube_url diff --git a/app/tests/unit/test_utils.py b/app/tests/unit/test_utils.py index 88fec7d3..1214898c 100644 --- a/app/tests/unit/test_utils.py +++ b/app/tests/unit/test_utils.py @@ -1,3 +1,19 @@ +# ContentDB +# Copyright (C) rubenwardy +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + import user_agents diff --git a/app/utils/__init__.py b/app/utils/__init__.py index bf018f24..af82dd0d 100644 --- a/app/utils/__init__.py +++ b/app/utils/__init__.py @@ -18,14 +18,12 @@ import re import secrets from typing import Dict -import typing - import deep_compare +from flask import current_app from .flask import * from .models import * from .user import * -from flask import current_app YESES = ["yes", "true", "1", "on"] diff --git a/app/utils/flask.py b/app/utils/flask.py index 2651ecd8..3364dd20 100644 --- a/app/utils/flask.py +++ b/app/utils/flask.py @@ -14,15 +14,16 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . +import datetime import typing from urllib.parse import urljoin, urlparse, urlunparse import user_agents -from flask import request, abort -from flask_babel import LazyString +from flask import request, abort, url_for +from flask_babel import LazyString, lazy_gettext from werkzeug.datastructures import MultiDict -from app.models import * +from app import app def is_safe_url(target): @@ -136,7 +137,7 @@ def get_request_date(key: str) -> typing.Optional[datetime.date]: abort(400) -def get_daterange_options() -> List[Tuple[LazyString, str]]: +def get_daterange_options() -> typing.List[typing.Tuple[LazyString, str]]: now = datetime.datetime.utcnow().date() days7 = (datetime.datetime.utcnow() - datetime.timedelta(days=7)).date() days30 = (datetime.datetime.utcnow() - datetime.timedelta(days=30)).date() diff --git a/app/utils/git.py b/app/utils/git.py index 9475ce85..12838854 100644 --- a/app/utils/git.py +++ b/app/utils/git.py @@ -15,8 +15,14 @@ # along with this program. If not, see . -import contextlib, git, gitdb, os, shutil, tempfile +import contextlib +import git +import gitdb +import os +import shutil +import tempfile from urllib.parse import urlsplit + from git import GitCommandError from app.tasks import TaskError diff --git a/app/utils/minetest_hypertext.py b/app/utils/minetest_hypertext.py index ca2db02e..280c1e5c 100644 --- a/app/utils/minetest_hypertext.py +++ b/app/utils/minetest_hypertext.py @@ -1,3 +1,19 @@ +# ContentDB +# Copyright (C) rubenwardy +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + from html.parser import HTMLParser import re import sys diff --git a/app/utils/phpbbparser.py b/app/utils/phpbbparser.py index 420dac52..0622bd6e 100644 --- a/app/utils/phpbbparser.py +++ b/app/utils/phpbbparser.py @@ -8,7 +8,8 @@ import urllib.parse as urlparse import urllib.request from datetime import datetime from urllib.parse import urlencode -from bs4 import * + +from bs4 import BeautifulSoup def urlEncodeNonAscii(b): diff --git a/app/utils/user.py b/app/utils/user.py index 05768310..dcf6fba3 100644 --- a/app/utils/user.py +++ b/app/utils/user.py @@ -22,8 +22,8 @@ from flask_login import login_user, current_user from passlib.handlers.bcrypt import bcrypt from flask import redirect, url_for, abort, flash -from app.models import User, UserRank, UserNotificationPreferences, db from app.utils import is_safe_url +from app.models import User, UserRank, UserNotificationPreferences, db def check_password_hash(stored, given):