Clean up code

This commit is contained in:
rubenwardy 2020-12-04 02:23:04 +00:00
parent 0c0d3e1715
commit 42f96618e2
62 changed files with 254 additions and 558 deletions

@ -15,18 +15,20 @@
# along with this program. If not, see <https://www.gnu.org/licenses/>.
import os
from celery import group
from flask import *
from flask_user import *
import flask_menu as menu
from . import bp
from app.models import *
from celery import uuid, group
from app.tasks.importtasks import importRepoScreenshot, makeVCSRelease, checkZipRelease, updateMetaFromRelease, importForeignDownloads
from app.tasks.forumtasks import importTopicList, checkAllForumAccounts
from flask_wtf import FlaskForm
from wtforms import *
from app.utils import loginUser, rank_required, addNotification
import datetime, os
from app.models import *
from app.tasks.forumtasks import importTopicList, checkAllForumAccounts
from app.tasks.importtasks import importRepoScreenshot, checkZipRelease, updateMetaFromRelease, importForeignDownloads
from app.utils import loginUser, rank_required
from . import bp
@bp.route("/admin/", methods=["GET", "POST"])
@rank_required(UserRank.ADMIN)

@ -16,13 +16,14 @@
from flask import *
from flask_user import *
from . import bp
from app.models import *
from flask_wtf import FlaskForm
from wtforms import *
from wtforms.validators import *
from app.models import *
from app.utils import rank_required
from . import bp
@bp.route("/licenses/")
@rank_required(UserRank.MODERATOR)

@ -17,12 +17,13 @@
from flask import *
from flask_user import *
from . import bp
from app.models import *
from flask_wtf import FlaskForm
from wtforms import *
from wtforms.validators import *
from app.utils import rank_required
from app.models import *
from . import bp
@bp.route("/tags/")
@login_required

@ -16,13 +16,14 @@
from flask import *
from flask_user import *
from . import bp
from app.models import *
from flask_wtf import FlaskForm
from wtforms import *
from wtforms.validators import *
from app.models import *
from app.utils import rank_required
from . import bp
@bp.route("/versions/")
@rank_required(UserRank.MODERATOR)

@ -16,13 +16,14 @@
from flask import *
from flask_user import *
from . import bp
from app.models import *
from flask_wtf import FlaskForm
from wtforms import *
from wtforms.validators import *
from app.models import *
from app.utils import rank_required
from . import bp
@bp.route("/admin/warnings/")
@rank_required(UserRank.ADMIN)

@ -14,10 +14,13 @@
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <https://www.gnu.org/licenses/>.
from flask import request, make_response, jsonify, abort
from functools import wraps
from flask import request, abort
from app.models import APIToken
from .support import error
from functools import wraps
def is_api_authd(f):
@wraps(f)

@ -78,7 +78,7 @@ def resolve_package_deps(out, package, only_hard):
# TODO: resolve most likely candidate
else:
raise "Malformed dependency"
raise Exception("Malformed dependency")
ret.append({
"name": name,

@ -17,19 +17,19 @@
from flask import render_template, redirect, request, session, url_for, abort
from flask_user import login_required, current_user
from . import bp
from app.models import db, User, APIToken, Package, Permission
from app.utils import randomString
from app.querybuilder import QueryBuilder
from flask_wtf import FlaskForm
from wtforms import *
from wtforms.validators import *
from wtforms.ext.sqlalchemy.fields import QuerySelectField
from wtforms.validators import *
from app.models import db, User, APIToken, Package, Permission
from app.utils import randomString
from . import bp
class CreateAPIToken(FlaskForm):
name = StringField("Name", [InputRequired(), Length(1, 30)])
package = QuerySelectField("Limit to package", allow_blank=True, \
package = QuerySelectField("Limit to package", allow_blank=True,
get_pk=lambda a: a.id, get_label=lambda a: a.title)
submit = SubmitField("Save")

@ -21,7 +21,6 @@ bp = Blueprint("github", __name__)
from flask import redirect, url_for, request, flash, abort, render_template, jsonify, current_app
from flask_user import current_user, login_required
from sqlalchemy import func, or_, and_
from flask_github import GitHub
from app import github, csrf
from app.models import db, User, APIToken, Package, Permission
from app.utils import loginUser, randomString, abs_url_for
@ -188,7 +187,7 @@ def setup_webhook():
return redirect(package.getDetailsURL())
if current_user.github_access_token is None:
return github.authorize("write:repo_hook", \
return github.authorize("write:repo_hook",
redirect_uri=abs_url_for("github.callback_webhook", pid=pid))
form = SetupWebhookForm(formdata=request.form)
@ -203,14 +202,14 @@ def setup_webhook():
if event != "push" and event != "create":
abort(500)
if handleMakeWebhook(gh_user, gh_repo, package, \
if handleMakeWebhook(gh_user, gh_repo, package,
current_user.github_access_token, event, token):
flash("Successfully created webhook", "success")
return redirect(package.getDetailsURL())
else:
return redirect(url_for("github.setup_webhook", pid=package.id))
return render_template("github/setup_webhook.html", \
return render_template("github/setup_webhook.html",
form=form, package=package)

@ -11,8 +11,8 @@ from sqlalchemy.sql.expression import func
@menu.register_menu(bp, ".", "Home")
def home():
def join(query):
return query.options( \
joinedload(Package.license), \
return query.options(
joinedload(Package.license),
joinedload(Package.media_license))
query = Package.query.filter_by(state=PackageState.APPROVED)
@ -37,5 +37,5 @@ def home():
tags = db.session.query(func.count(Tags.c.tag_id), Tag) \
.select_from(Tag).outerjoin(Tags).group_by(Tag.id).order_by(db.asc(Tag.title)).all()
return render_template("index.html", count=count, downloads=downloads, tags=tags, \
return render_template("index.html", count=count, downloads=downloads, tags=tags,
new=new, updated=updated, pop_mod=pop_mod, pop_txp=pop_txp, pop_gam=pop_gam, reviews=reviews)

@ -19,7 +19,6 @@ from flask import *
bp = Blueprint("metapackages", __name__)
from flask_user import *
from app.models import *
@bp.route("/metapackages/")
@ -59,6 +58,6 @@ def view(name):
.order_by(db.asc(ForumTopic.name), db.asc(ForumTopic.title)) \
.all()
return render_template("metapackages/view.html", mpackage=mpackage, \
dependers=dependers, optional_dependers=optional_dependers, \
return render_template("metapackages/view.html", mpackage=mpackage,
dependers=dependers, optional_dependers=optional_dependers,
similar_topics=similar_topics)

@ -15,9 +15,10 @@
# along with this program. If not, see <https://www.gnu.org/licenses/>.
from flask import Blueprint, make_response
from app.models import Package, PackageRelease, db, User, UserRank, PackageState
from sqlalchemy.sql.expression import func
from app.models import Package, db, User, UserRank, PackageState
bp = Blueprint("metrics", __name__)
def generate_metrics(full=False):
@ -28,16 +29,16 @@ def generate_metrics(full=False):
def gen_labels(labels):
pieces = [key + "=" + str(val) for key, val in labels.items()]
return (",").join(pieces)
return ",".join(pieces)
def write_array_stat(name, help, type, data):
ret = ("# HELP {name} {help}\n# TYPE {name} {type}\n") \
ret = "# HELP {name} {help}\n# TYPE {name} {type}\n" \
.format(name=name, help=help, type=type)
for entry in data:
assert(len(entry) == 2)
ret += ("{name}{{{labels}}} {value}\n") \
ret += "{name}{{{labels}}} {value}\n" \
.format(name=name, labels=gen_labels(entry[0]), value=entry[1])
return ret + "\n"
@ -57,7 +58,7 @@ def generate_metrics(full=False):
scores = Package.query.join(User).with_entities(User.username, Package.name, Package.score) \
.filter(Package.state==PackageState.APPROVED).all()
ret += write_array_stat("contentdb_package_score", "Package score", "gauge", \
ret += write_array_stat("contentdb_package_score", "Package score", "gauge",
[({ "author": score[0], "name": score[1] }, score[2]) for score in scores])
else:
score_result = db.session.query(func.sum(Package.score)).one_or_none()

@ -1,172 +0,0 @@
# ContentDB
# Copyright (C) 2018 rubenwardy
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <https://www.gnu.org/licenses/>.
from flask import *
from flask_user import *
from app import app
from app.models import *
from app.utils import *
from flask_wtf import FlaskForm
from wtforms import *
from wtforms.validators import *
from wtforms.ext.sqlalchemy.fields import QuerySelectField, QuerySelectMultipleField
from . import PackageForm
class EditRequestForm(PackageForm):
edit_title = StringField("Edit Title", [InputRequired(), Length(1, 100)])
edit_desc = TextField("Edit Description", [Optional()])
@app.route("/packages/<author>/<name>/requests/new/", methods=["GET","POST"])
@app.route("/packages/<author>/<name>/requests/<id>/edit/", methods=["GET","POST"])
@login_required
@is_package_page
def create_edit_editrequest_page(package, id=None):
edited_package = package
erequest = None
if id is not None:
erequest = EditRequest.query.get(id)
if erequest.package != package:
abort(404)
if not erequest.checkPerm(current_user, Permission.EDIT_EDITREQUEST):
abort(403)
if erequest.status != 0:
flash("Can't edit EditRequest, it has already been merged or rejected", "danger")
return redirect(erequest.getURL())
edited_package = Package(package)
erequest.applyAll(edited_package)
form = EditRequestForm(request.form, obj=edited_package)
if request.method == "GET":
deps = edited_package.dependencies
form.harddep_str.data = ",".join([str(x) for x in deps if not x.optional])
form.softdep_str.data = ",".join([str(x) for x in deps if x.optional])
form.provides_str.data = MetaPackage.ListToSpec(edited_package.provides)
if request.method == "POST" and form.validate():
if erequest is None:
erequest = EditRequest()
erequest.package = package
erequest.author = current_user
erequest.title = form["edit_title"].data
erequest.desc = form["edit_desc"].data
db.session.add(erequest)
EditRequestChange.query.filter_by(request=erequest).delete()
wasChangeMade = False
for e in PackagePropertyKey:
newValue = form[e.name].data
oldValue = getattr(package, e.name)
newValueComp = newValue
oldValueComp = oldValue
if type(newValue) is str:
newValue = newValue.replace("\r\n", "\n")
newValueComp = newValue.strip()
oldValueComp = "" if oldValue is None else oldValue.strip()
if newValueComp != oldValueComp:
change = EditRequestChange()
change.request = erequest
change.key = e
change.oldValue = e.convert(oldValue)
change.newValue = e.convert(newValue)
db.session.add(change)
wasChangeMade = True
if wasChangeMade:
msg = "Edit request #{} {}" \
.format(erequest.id, "created" if id is None else "edited")
addNotification(package.maintainers, current_user, msg, erequest.getURL(), package)
addNotification(erequest.author, current_user, msg, erequest.getURL(), package)
db.session.commit()
return redirect(erequest.getURL())
else:
flash("No changes detected", "warning")
elif erequest is not None:
form["edit_title"].data = erequest.title
form["edit_desc"].data = erequest.desc
return render_template("packages/editrequest_create_edit.html", package=package, form=form)
@app.route("/packages/<author>/<name>/requests/<id>/")
@is_package_page
def view_editrequest_page(package, id):
erequest = EditRequest.query.get(id)
if erequest is None or erequest.package != package:
abort(404)
return render_template("packages/editrequest_view.html", package=package, request=erequest)
@app.route("/packages/<author>/<name>/requests/<id>/approve/", methods=["POST"])
@is_package_page
def approve_editrequest_page(package, id):
if not package.checkPerm(current_user, Permission.APPROVE_CHANGES):
flash("You don't have permission to do that.", "danger")
return redirect(package.getDetailsURL())
erequest = EditRequest.query.get(id)
if erequest is None or erequest.package != package:
abort(404)
if erequest.status != 0:
flash("Edit request has already been resolved", "danger")
else:
erequest.status = 1
erequest.applyAll(package)
msg = "Edit request #{} merged".format(erequest.id)
addNotification(erequest.author, current_user, msg, erequest.getURL(), package)
addNotification(package.maintainers, current_user, msg, erequest.getURL(), package)
db.session.commit()
return redirect(package.getDetailsURL())
@app.route("/packages/<author>/<name>/requests/<id>/reject/", methods=["POST"])
@is_package_page
def reject_editrequest_page(package, id):
if not package.checkPerm(current_user, Permission.APPROVE_CHANGES):
flash("You don't have permission to do that.", "danger")
return redirect(package.getDetailsURL())
erequest = EditRequest.query.get(id)
if erequest is None or erequest.package != package:
abort(404)
if erequest.status != 0:
flash("Edit request has already been resolved", "danger")
else:
erequest.status = 2
msg = "Edit request #{} rejected".format(erequest.id)
addNotification(erequest.author, current_user, msg, erequest.getURL(), package)
addNotification(package.maintainers, current_user, msg, erequest.getURL(), package)
db.session.commit()
return redirect(package.getDetailsURL())

@ -15,27 +15,23 @@
# along with this program. If not, see <https://www.gnu.org/licenses/>.
from flask import render_template, abort, request, redirect, url_for, flash
from flask_user import current_user
import flask_menu as menu
from . import bp
from app.models import *
from app.querybuilder import QueryBuilder
from app.tasks.importtasks import importRepoScreenshot, updateMetaFromRelease
from app.rediscache import has_key, set_key
from app.utils import *
from flask_wtf import FlaskForm
from wtforms import *
from wtforms.validators import *
from wtforms.ext.sqlalchemy.fields import QuerySelectField, QuerySelectMultipleField
from sqlalchemy import or_, func
from sqlalchemy.orm import joinedload, subqueryload
from urllib.parse import quote as urlescape
import flask_menu as menu
from celery import uuid
from flask import render_template
from flask_wtf import FlaskForm
from sqlalchemy import or_, func
from sqlalchemy.orm import joinedload, subqueryload
from wtforms import *
from wtforms.ext.sqlalchemy.fields import QuerySelectField, QuerySelectMultipleField
from wtforms.validators import *
from app.querybuilder import QueryBuilder
from app.rediscache import has_key, set_key
from app.tasks.importtasks import importRepoScreenshot, updateMetaFromRelease
from app.utils import *
from . import bp
@menu.register_menu(bp, ".mods", "Mods", order=11, endpoint_arguments_constructor=lambda: { 'type': 'mod' })
@ -48,9 +44,9 @@ def list_all():
query = qb.buildPackageQuery()
title = qb.title
query = query.options( \
joinedload(Package.license), \
joinedload(Package.media_license), \
query = query.options(
joinedload(Package.license),
joinedload(Package.media_license),
subqueryload(Package.tags))
ip = request.headers.get("X-Forwarded-For") or request.remote_addr
@ -103,9 +99,9 @@ def list_all():
selected_tags = set(qb.tags)
return render_template("packages/list.html", \
title=title, packages=query.items, pagination=query, \
query=search, tags=tags, selected_tags=selected_tags, type=type_name, \
return render_template("packages/list.html",
title=title, packages=query.items, pagination=query,
query=search, tags=tags, selected_tags=selected_tags, type=type_name,
authors=authors, packages_count=query.total, topics=topics)
@ -176,10 +172,10 @@ def view(package):
has_review = current_user.is_authenticated and PackageReview.query.filter_by(package=package, author=current_user).count() > 0
return render_template("packages/view.html", \
package=package, releases=releases, requests=requests, \
alternatives=alternatives, similar_topics=similar_topics, \
review_thread=review_thread, topic_error=topic_error, topic_error_lvl=topic_error_lvl, \
return render_template("packages/view.html",
package=package, releases=releases, requests=requests,
alternatives=alternatives, similar_topics=similar_topics,
review_thread=review_thread, topic_error=topic_error, topic_error_lvl=topic_error_lvl,
threads=threads.all(), has_review=has_review)
@ -366,9 +362,9 @@ def create_edit(author=None, name=None):
package_query = package_query.filter(Package.id != package.id)
enableWizard = name is None and request.method != "POST"
return render_template("packages/create_edit.html", package=package, \
form=form, author=author, enable_wizard=enableWizard, \
packages=package_query.all(), \
return render_template("packages/create_edit.html", package=package,
form=form, author=author, enable_wizard=enableWizard,
packages=package_query.all(),
mpackages=MetaPackage.query.order_by(db.asc(MetaPackage.name)).all())
@ -504,7 +500,7 @@ def edit_maintainers(package):
users = User.query.filter(User.rank >= UserRank.NEW_MEMBER).order_by(db.asc(User.username)).all()
return render_template("packages/edit_maintainers.html", \
return render_template("packages/edit_maintainers.html",
package=package, form=form, users=users)

@ -15,21 +15,17 @@
# along with this program. If not, see <https://www.gnu.org/licenses/>.
from flask import *
from flask_user import *
from . import bp
from app.rediscache import has_key, set_key, make_download_key
from app.models import *
from app.tasks.importtasks import makeVCSRelease, checkZipRelease, updateMetaFromRelease
from app.utils import *
from celery import uuid
from flask import *
from flask_wtf import FlaskForm
from wtforms import *
from wtforms.validators import *
from wtforms.ext.sqlalchemy.fields import QuerySelectField
from wtforms.validators import *
from app.rediscache import has_key, set_key, make_download_key
from app.tasks.importtasks import makeVCSRelease, checkZipRelease, updateMetaFromRelease
from app.utils import *
from . import bp
def get_mt_releases(is_max):

@ -110,7 +110,7 @@ def review(package):
return redirect(package.getDetailsURL())
return render_template("packages/review_create_edit.html", \
return render_template("packages/review_create_edit.html",
form=form, package=package, review=review)

@ -16,17 +16,13 @@
from flask import *
from flask_user import *
from . import bp
from app.models import *
from app.utils import *
from flask_wtf import FlaskForm
from wtforms import *
from wtforms.validators import *
from app.utils import *
from . import bp
class CreateScreenshotForm(FlaskForm):
title = StringField("Title/Caption", [Optional(), Length(-1, 100)])
@ -43,7 +39,7 @@ class EditScreenshotForm(FlaskForm):
@bp.route("/packages/<author>/<name>/screenshots/new/", methods=["GET", "POST"])
@login_required
@is_package_page
def create_screenshot(package, id=None):
def create_screenshot(package):
if not package.checkPerm(current_user, Permission.ADD_SCREENSHOTS):
return redirect(package.getDetailsURL())

@ -16,13 +16,10 @@
from flask import *
from flask_user import *
import flask_menu as menu
from app import csrf
from app.models import *
from app.tasks import celery, TaskError
from app.tasks import celery
from app.tasks.importtasks import getMeta
from app.utils import shouldReturnJson
from app.utils import *
bp = Blueprint("tasks", __name__)
@ -45,7 +42,7 @@ def check(id):
traceback = result.traceback
result = result.result
info = None
None
if isinstance(result, Exception):
info = {
'id': id,

@ -21,9 +21,7 @@ bp = Blueprint("threads", __name__)
from flask_user import *
from app.models import *
from app.utils import addNotification, clearNotifications, isYes, addAuditLog
import datetime
from app.utils import addNotification, isYes, addAuditLog
from flask_wtf import FlaskForm
from wtforms import *
@ -199,7 +197,7 @@ def view(id):
flash("Please wait before commenting again", "danger")
return redirect(thread.getViewURL())
if len(comment) <= 2000 and len(comment) > 3:
if 2000 >= len(comment) > 3:
reply = ThreadReply()
reply.author = current_user
reply.comment = comment

@ -16,11 +16,11 @@
from flask import *
from flask_user import *
import flask_menu as menu
from sqlalchemy import or_
from app.models import *
from app.querybuilder import QueryBuilder
from app.utils import get_int_or_abort
from sqlalchemy import or_
bp = Blueprint("todo", __name__)
@ -80,8 +80,8 @@ def view():
return render_template("todo/list.html", title="Reports and Work Queue",
packages=packages, wip_packages=wip_packages, releases=releases, screenshots=screenshots,
canApproveNew=canApproveNew, canApproveRel=canApproveRel, canApproveScn=canApproveScn,
topics_to_add=topics_to_add, total_topics=total_topics, \
total_packages=total_packages, total_to_tag=total_to_tag, \
topics_to_add=topics_to_add, total_topics=total_topics,
total_packages=total_packages, total_to_tag=total_to_tag,
unfulfilled_meta_packages=unfulfilled_meta_packages)
@ -104,16 +104,16 @@ def topics():
num = 100
query = query.paginate(page, num, True)
next_url = url_for("todo.topics", page=query.next_num, query=qb.search, \
next_url = url_for("todo.topics", page=query.next_num, query=qb.search,
show_discarded=qb.show_discarded, n=num, sort=qb.order_by) \
if query.has_next else None
prev_url = url_for("todo.topics", page=query.prev_num, query=qb.search, \
prev_url = url_for("todo.topics", page=query.prev_num, query=qb.search,
show_discarded=qb.show_discarded, n=num, sort=qb.order_by) \
if query.has_prev else None
return render_template("todo/topics.html", topics=query.items, total=total, \
topic_count=topic_count, query=qb.search, show_discarded=qb.show_discarded, \
next_url=next_url, prev_url=prev_url, page=page, page_max=query.pages, \
return render_template("todo/topics.html", topics=query.items, total=total,
topic_count=topic_count, query=qb.search, show_discarded=qb.show_discarded,
next_url=next_url, prev_url=prev_url, page=page, page_max=query.pages,
n=num, sort_by=qb.order_by)

@ -16,19 +16,19 @@
from flask import *
from flask_user import signals, current_user, user_manager
from flask_login import login_user, logout_user
from app.markdown import render_markdown
from . import bp
from app.models import *
from flask_user import signals, current_user, user_manager, login_required
from flask_wtf import FlaskForm
from sqlalchemy import func
from wtforms import *
from wtforms.validators import *
from app.utils import randomString, loginUser, rank_required, nonEmptyOrNone, addAuditLog
from app.tasks.forumtasks import checkForumAccount
from app.markdown import render_markdown
from app.models import *
from app.tasks.emails import sendVerifyEmail, sendEmailRaw
from app.tasks.phpbbparser import getProfile
from sqlalchemy import func
from app.tasks.forumtasks import checkForumAccount
from app.utils import randomString, rank_required, nonEmptyOrNone, addAuditLog
from . import bp
# Define the User profile form
class UserProfileForm(FlaskForm):
@ -198,7 +198,7 @@ def set_password():
return redirect(url_for("user.change_password"))
form = SetPasswordForm(request.form)
if current_user.email == None:
if current_user.email is None:
form.email.validators = [InputRequired(), Email()]
if request.method == "POST" and form.validate():

@ -17,9 +17,9 @@ def populate(session):
session.add(MinetestRelease("5.1", 38))
tags = {}
for tag in ["Inventory", "Mapgen", "Building", \
"Mobs and NPCs", "Tools", "Player effects", \
"Environment", "Transport", "Maintenance", "Plants and farming", \
for tag in ["Inventory", "Mapgen", "Building",
"Mobs and NPCs", "Tools", "Player effects",
"Environment", "Transport", "Maintenance", "Plants and farming",
"PvP", "PvE", "Survival", "Creative", "Puzzle", "Multiplayer", "Singleplayer"]:
row = Tag(tag)
tags[row.name] = row

@ -1,7 +1,8 @@
import logging
from enum import Enum
from app.tasks.emails import sendEmailRaw
def _has_newline(line):
"""Used by has_bad_header to check for \\r or \\n"""
if line and ("\r" in line or "\n" in line):

@ -53,8 +53,8 @@ ALLOWED_PROTOCOLS = ['http', 'https', 'mailto']
md = Markdown(extensions=["fenced_code"], output_format="html5")
def render_markdown(source):
return bleach.clean(md.convert(source), \
tags=ALLOWED_TAGS, attributes=ALLOWED_ATTRIBUTES, \
return bleach.clean(md.convert(source),
tags=ALLOWED_TAGS, attributes=ALLOWED_ATTRIBUTES,
styles=[], protocols=ALLOWED_PROTOCOLS)
def init_app(app):

@ -15,20 +15,18 @@
# along with this program. If not, see <https://www.gnu.org/licenses/>.
import enum, datetime
from app import app, gravatar
import datetime
import enum
from urllib.parse import urlparse
from flask import Flask, url_for
from flask_sqlalchemy import SQLAlchemy, BaseQuery
from flask import url_for
from flask_migrate import Migrate
from flask_user import login_required, UserManager, UserMixin
from sqlalchemy import func, CheckConstraint
from sqlalchemy_searchable import SearchQueryMixin
from flask_sqlalchemy import SQLAlchemy, BaseQuery
from flask_user import UserManager, UserMixin
from sqlalchemy_searchable import SearchQueryMixin, make_searchable
from sqlalchemy_utils.types import TSVectorType
from sqlalchemy_searchable import make_searchable
from app import app, gravatar
# Initialise database
db = SQLAlchemy(app)
@ -163,7 +161,7 @@ class User(db.Model, UserMixin):
donate_url = db.Column(db.String(255), nullable=True, default=None)
# Content
notifications = db.relationship("Notification", primaryjoin="User.id==Notification.user_id", \
notifications = db.relationship("Notification", primaryjoin="User.id==Notification.user_id",
order_by="Notification.created_at")
packages = db.relationship("Package", backref=db.backref("author", lazy="joined"), lazy="dynamic")
@ -377,11 +375,11 @@ class PackageState(enum.Enum):
PACKAGE_STATE_FLOW = {
PackageState.WIP: set([ PackageState.READY_FOR_REVIEW ]),
PackageState.CHANGES_NEEDED: set([ PackageState.READY_FOR_REVIEW ]),
PackageState.READY_FOR_REVIEW: set([ PackageState.WIP, PackageState.CHANGES_NEEDED, PackageState.APPROVED ]),
PackageState.APPROVED: set([ PackageState.CHANGES_NEEDED ]),
PackageState.DELETED: set([ PackageState.READY_FOR_REVIEW ]),
PackageState.WIP: {PackageState.READY_FOR_REVIEW},
PackageState.CHANGES_NEEDED: {PackageState.READY_FOR_REVIEW},
PackageState.READY_FOR_REVIEW: {PackageState.WIP, PackageState.CHANGES_NEEDED, PackageState.APPROVED},
PackageState.APPROVED: {PackageState.CHANGES_NEEDED},
PackageState.DELETED: {PackageState.READY_FOR_REVIEW},
}
@ -471,7 +469,7 @@ class Dependency(db.Model):
raise Exception("Meta and package are both none!")
@staticmethod
def SpecToList(depender, spec, cache={}):
def SpecToList(depender, spec, cache):
retval = []
arr = spec.split(",")
@ -521,7 +519,7 @@ class Package(db.Model):
name_valid = db.CheckConstraint("name ~* '^[a-z0-9_]+$'")
search_vector = db.Column(TSVectorType("name", "title", "short_desc", "desc", \
search_vector = db.Column(TSVectorType("name", "title", "short_desc", "desc",
weights={ "name": "A", "title": "B", "short_desc": "C", "desc": "D" }))
license_id = db.Column(db.Integer, db.ForeignKey("license.id"), nullable=False, default=1)
@ -548,8 +546,8 @@ class Package(db.Model):
issueTracker = db.Column(db.String(200), nullable=True)
forums = db.Column(db.Integer, nullable=True)
provides = db.relationship("MetaPackage", \
secondary=provides, lazy="select", order_by=db.asc("name"), \
provides = db.relationship("MetaPackage",
secondary=provides, lazy="select", order_by=db.asc("name"),
backref=db.backref("packages", lazy="dynamic", order_by=db.desc("score")))
dependencies = db.relationship("Dependency", backref="depender", lazy="dynamic", foreign_keys=[Dependency.depender_id])
@ -613,7 +611,7 @@ class Package(db.Model):
user = m.group(1)
repo = m.group(2).replace(".git", "")
return (user,repo)
return user, repo
def getSortedDependencies(self, is_hard=None):
query = self.dependencies
@ -770,7 +768,7 @@ class Package(db.Model):
def getDownloadRelease(self, version=None):
for rel in self.releases:
if rel.approved and (version is None or
((rel.min_rel is None or rel.min_rel_id <= version.id) and \
((rel.min_rel is None or rel.min_rel_id <= version.id) and
(rel.max_rel is None or rel.max_rel_id >= version.id))):
return rel
@ -916,7 +914,7 @@ class MetaPackage(db.Model):
return ",".join([str(x) for x in list])
@staticmethod
def GetOrCreate(name, cache={}):
def GetOrCreate(name, cache):
mp = cache.get(name)
if mp is None:
mp = MetaPackage.query.filter_by(name=name).first()
@ -929,7 +927,7 @@ class MetaPackage(db.Model):
return mp
@staticmethod
def SpecToList(spec, cache={}):
def SpecToList(spec, cache):
retval = []
arr = spec.split(",")
@ -1131,7 +1129,7 @@ class PackageScreenshot(db.Model):
id=self.id)
def getThumbnailURL(self, level=2):
return self.url.replace("/uploads/", ("/thumbnails/{:d}/").format(level))
return self.url.replace("/uploads/", "/thumbnails/{:d}/".format(level))
class APIToken(db.Model):
@ -1264,10 +1262,10 @@ class Thread(db.Model):
created_at = db.Column(db.DateTime, nullable=False, default=datetime.datetime.utcnow)
replies = db.relationship("ThreadReply", backref="thread", lazy="dynamic", \
replies = db.relationship("ThreadReply", backref="thread", lazy="dynamic",
order_by=db.asc("thread_reply_id"))
watchers = db.relationship("User", secondary=watchers, lazy="subquery", \
watchers = db.relationship("User", secondary=watchers, lazy="subquery",
backref=db.backref("watching", lazy=True))
def getViewURL(self):
@ -1412,9 +1410,9 @@ class AuditLogEntry(db.Model):
REPO_BLACKLIST = [".zip", "mediafire.com", "dropbox.com", "weebly.com", \
"minetest.net", "dropboxusercontent.com", "4shared.com", \
"digitalaudioconcepts.com", "hg.intevation.org", "www.wtfpl.net", \
REPO_BLACKLIST = [".zip", "mediafire.com", "dropbox.com", "weebly.com",
"minetest.net", "dropboxusercontent.com", "4shared.com",
"digitalaudioconcepts.com", "hg.intevation.org", "www.wtfpl.net",
"imageshack.com", "imgur.com"]
class ForumTopic(db.Model):

@ -1,8 +1,10 @@
from .models import db, PackageType, Package, ForumTopic, License, MinetestRelease, PackageRelease, User, Tag, Tags, ContentWarning, PackageState
from .utils import isNo, isYes, get_int_or_abort
from sqlalchemy.sql.expression import func
from flask import abort
from sqlalchemy import or_
from sqlalchemy.sql.expression import func
from .models import db, PackageType, Package, ForumTopic, License, MinetestRelease, PackageRelease, User, Tag, ContentWarning, PackageState
from .utils import isYes, get_int_or_abort
class QueryBuilder:
title = None

@ -4,7 +4,7 @@ from . import r
# and also means that the releases code avoids knowing about `app`
def make_download_key(ip, package):
return ("{}/{}/{}").format(ip, package.author.username, package.name)
return "{}/{}/{}".format(ip, package.author.username, package.name)
def set_key(key, v):
r.set(key, v)

@ -53,11 +53,11 @@ def sass(app, inputDir='scss', outputPath='static', force=False, cacheDir="publi
cacheFile = "%s/%s.css" % (cacheDir, filepath)
# Source file exists, and needs regenerating
if os.path.isfile(sassfile) and (force or not os.path.isfile(cacheFile) or \
if os.path.isfile(sassfile) and (force or not os.path.isfile(cacheFile) or
os.path.getmtime(sassfile) > os.path.getmtime(cacheFile)):
_convert(inputDir, sassfile, cacheFile)
app.logger.debug('Compiled %s into %s' % (sassfile, cacheFile))
return send_from_directory(cacheDir, filepath + ".css")
app.add_url_rule("/%s/<path:filepath>.css" % (outputPath), 'sass', _sass)
app.add_url_rule("/%s/<path:filepath>.css" % outputPath, 'sass', _sass)

@ -16,10 +16,8 @@
import flask
from flask_sqlalchemy import SQLAlchemy
from celery import Celery
from celery.schedules import crontab
from app import app
from app.models import *
class TaskError(Exception):

@ -15,7 +15,7 @@
# along with this program. If not, see <https://www.gnu.org/licenses/>.
from flask import render_template, url_for
from flask import render_template
from flask_mail import Message
from app import mail
from app.tasks import celery

@ -15,14 +15,11 @@
# along with this program. If not, see <https://www.gnu.org/licenses/>.
import flask, json, re
from flask_sqlalchemy import SQLAlchemy
from app import app
import json, re
from app.models import *
from app.tasks import celery
from .phpbbparser import getProfile, getTopicsFromForum
import urllib.request
from urllib.parse import urlparse, quote_plus
@celery.task()
def checkForumAccount(username, forceNoSave=False):

@ -15,22 +15,17 @@
# along with this program. If not, see <https://www.gnu.org/licenses/>.
import flask, json, os, git, tempfile, shutil, gitdb, contextlib
import os, git, tempfile, shutil, gitdb, contextlib
from git import GitCommandError
from git_archive_all import GitArchiver
from flask_sqlalchemy import SQLAlchemy
from urllib.error import HTTPError
import urllib.request
from urllib.parse import urlparse, quote_plus, urlsplit
from urllib.parse import urlsplit
from zipfile import ZipFile
from app import app
from app.models import *
from app.tasks import celery, TaskError
from app.utils import randomString, getExtension
from .minetestcheck import build_tree, MinetestCheckError, ContentType
from .minetestcheck.config import parse_conf
from .krocklist import getKrockList, findModInfo
def generateGitURL(urlstr):
@ -60,7 +55,7 @@ def cloneRepo(urlstr, ref=None, recursive=False):
print("Cloning from " + gitUrl)
if ref is None:
repo = git.Repo.clone_from(gitUrl, gitDir, \
repo = git.Repo.clone_from(gitUrl, gitDir,
progress=None, env=None, depth=1, recursive=recursive, kill_after_timeout=15)
else:
repo = git.Repo.init(gitDir)
@ -96,10 +91,7 @@ def getMeta(urlstr, author):
except MinetestCheckError as err:
raise TaskError(str(err))
result = {}
result["name"] = tree.name
result["provides"] = tree.getModNames()
result["type"] = tree.type.name
result = {"name": tree.name, "provides": tree.getModNames(), "type": tree.type.name}
for key in ["depends", "optional_depends"]:
result[key] = tree.fold("meta", key)
@ -120,7 +112,7 @@ def getMeta(urlstr, author):
def postReleaseCheckUpdate(self, release, path):
try:
tree = build_tree(path, expected_type=ContentType[release.package.type.name], \
tree = build_tree(path, expected_type=ContentType[release.package.type.name],
author=release.package.author.username, name=release.package.name)
cache = {}

@ -1,80 +0,0 @@
# ContentDB
# Copyright (C) 2018-20 rubenwardy
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <https://www.gnu.org/licenses/>.
import json
import urllib.request
krock_list_cache = None
krock_list_cache_by_name = None
def getKrockList():
global krock_list_cache
global krock_list_cache_by_name
if krock_list_cache is None:
contents = urllib.request.urlopen("https://krock-works.uk.to/minetest/modList.php").read().decode("utf-8")
list = json.loads(contents)
def h(x):
if not ("title" in x and "author" in x and \
"topicId" in x and "link" in x and x["link"] != ""):
return False
import re
m = re.search(r"\[([A-Za-z0-9_]+)\]", x["title"])
if m is None:
return False
x["name"] = m.group(1)
return True
def g(x):
return {
"title": x["title"],
"author": x["author"],
"name": x["name"],
"topicId": x["topicId"],
"link": x["link"],
}
krock_list_cache = [g(x) for x in list if h(x)]
krock_list_cache_by_name = {}
for x in krock_list_cache:
if not x["name"] in krock_list_cache_by_name:
krock_list_cache_by_name[x["name"]] = []
krock_list_cache_by_name[x["name"]].append(x)
return krock_list_cache, krock_list_cache_by_name
def findModInfo(author, name, link):
list, lookup = getKrockList()
if name is not None and name in lookup:
if len(lookup[name]) == 1:
return lookup[name][0]
for x in lookup[name]:
if x["author"] == author:
return x
if link is not None and len(link) > 15:
for x in list:
if link in x["link"]:
return x
return None

@ -20,7 +20,7 @@ class ContentType(Enum):
"""
Whether or not `other` is an acceptable type for this
"""
assert(other)
assert other
if self == ContentType.MOD:
if not other.isModLike():
@ -40,7 +40,7 @@ def build_tree(path, expected_type=None, author=None, repo=None, name=None):
path = get_base_dir(path)
root = PackageTreeNode(path, "/", author=author, repo=repo, name=name)
assert(root)
assert root
if expected_type:
expected_type.validate_same(root.type)

@ -54,11 +54,11 @@ class PackageTreeNode:
if self.type == ContentType.GAME:
if not os.path.isdir(baseDir + "/mods"):
raise MinetestCheckError(("Game at {} does not have a mods/ folder").format(self.relative))
raise MinetestCheckError("Game at {} does not have a mods/ folder".format(self.relative))
self.add_children_from_mod_dir("mods")
elif self.type == ContentType.MOD:
if self.name and not basenamePattern.match(self.name):
raise MinetestCheckError(("Invalid base name for mod {} at {}, names must only contain a-z0-9_.") \
raise MinetestCheckError("Invalid base name for mod {} at {}, names must only contain a-z0-9_." \
.format(self.name, self.relative))
elif self.type == ContentType.MODPACK:
self.add_children_from_mod_dir(None)
@ -132,10 +132,11 @@ class PackageTreeNode:
for dep in result["depends"]:
if not basenamePattern.match(dep):
if " " in dep:
raise MinetestCheckError(("Invalid dependency name '{}' for mod at {}, did you forget a comma?") \
raise MinetestCheckError("Invalid dependency name '{}' for mod at {}, did you forget a comma?" \
.format(dep, self.relative))
else:
raise MinetestCheckError(("Invalid dependency name '{}' for mod at {}, names must only contain a-z0-9_.") \
raise MinetestCheckError(
"Invalid dependency name '{}' for mod at {}, names must only contain a-z0-9_." \
.format(dep, self.relative))
@ -177,11 +178,11 @@ class PackageTreeNode:
if not entry.startswith('.') and os.path.isdir(path):
child = PackageTreeNode(path, relative + entry + "/", name=entry)
if not child.type.isModLike():
raise MinetestCheckError(("Expecting mod or modpack, found {} at {} inside {}") \
raise MinetestCheckError("Expecting mod or modpack, found {} at {} inside {}" \
.format(child.type.value, child.relative, self.type.value))
if child.name is None:
raise MinetestCheckError(("Missing base name for mod at {}").format(self.relative))
raise MinetestCheckError("Missing base name for mod at {}".format(self.relative))
self.children.append(child)

@ -2,15 +2,14 @@
# License: MIT
# Source: https://github.com/rubenwardy/python_phpbb_parser
import urllib, socket
from bs4 import *
from urllib.parse import urljoin
from datetime import datetime
import urllib.request
import os.path
import time, re
import re
import urllib
import urllib.parse as urlparse
import urllib.request
from datetime import datetime
from urllib.parse import urlencode
from bs4 import *
def urlEncodeNonAscii(b):
return re.sub('[\x80-\xFF]', lambda c: '%%%02x' % ord(c.group(0)), b)
@ -68,7 +67,7 @@ def __extract_properties(profile, soup):
def __extract_signature(soup):
res = soup.find_all("div", class_="signature")
if (len(res) != 1):
if len(res) != 1:
return None
else:
return res[0]
@ -166,7 +165,7 @@ def parseForumListPage(id, page, out, extra=None):
return True
def getTopicsFromForum(id, out={}, extra=None):
def getTopicsFromForum(id, out, extra=None):
print("Fetching all topics from forum {}".format(id))
page = 0
while parseForumListPage(id, page, out, extra):

@ -1,9 +1,7 @@
import pytest
from app import app
from app.default_data import populate_test_data
from app.models import db, License, Tag, User, UserRank, Package, PackageState
from utils import client, recreate_db, parse_json
from utils import is_str, is_int, is_optional
from app.models import db, Package, PackageState
from utils import parse_json, is_str, is_int, is_optional
def validate_package_list(packages, strict=False):
valid_keys = {

@ -1,8 +1,6 @@
import pytest
from app import app
from app.default_data import populate_test_data
from app.models import db, License, Tag, User, UserRank
from utils import client, recreate_db
from app.models import db
def test_homepage_empty(client):
"""Start with a blank database."""

@ -15,15 +15,22 @@
# along with this program. If not, see <https://www.gnu.org/licenses/>.
from flask import request, flash, abort, redirect
from flask_user import *
from flask_login import login_user, logout_user
from .models import *
from . import app
import random, string, os, imghdr, user_agents
import imghdr
import os
import random
import string
import user_agents
from urllib.parse import urljoin
from flask import request, flash, abort, redirect
from flask_login import login_user
from flask_user import *
from werkzeug.datastructures import MultiDict
from . import app
from .models import *
# These are given to Jinja in template_filters.py
def abs_url_for(path, **kwargs):
@ -78,7 +85,7 @@ def getExtension(filename):
def isFilenameAllowed(filename, exts):
return getExtension(filename) in exts
ALLOWED_IMAGES = set(["jpeg", "png"])
ALLOWED_IMAGES = {"jpeg", "png"}
def isAllowedImage(data):
return imghdr.what(None, data) in ALLOWED_IMAGES

@ -5,10 +5,6 @@ Revises: e9f534df23a8
Create Date: 2018-06-03 01:47:33.006039
"""
from alembic import op
import sqlalchemy as sa
import sqlalchemy.types as ty
# revision identifiers, used by Alembic.
revision = '28a427cbd4cf'

@ -5,11 +5,10 @@ Revises: 9ec17b558413
Create Date: 2019-01-29 02:43:08.865695
"""
from alembic import op
import sqlalchemy as sa
from sqlalchemy.dialects import postgresql
from sqlalchemy_utils.types import TSVectorType
from alembic import op
from sqlalchemy_searchable import sync_trigger
from sqlalchemy_utils.types import TSVectorType
# revision identifiers, used by Alembic.
revision = '2f3c3597c78d'

@ -6,8 +6,6 @@ Create Date: 2020-01-18 23:00:40.487425
"""
from alembic import op
import sqlalchemy as sa
from sqlalchemy.dialects import postgresql
# revision identifiers, used by Alembic.
revision = '306ce331a2a7'

@ -6,8 +6,6 @@ Create Date: 2018-05-25 17:53:13.215127
"""
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision = '3f4d7cd8401f'

@ -6,8 +6,6 @@ Create Date: 2020-01-19 02:28:05.432244
"""
from alembic import op
import sqlalchemy as sa
from sqlalchemy.dialects import postgresql
# revision identifiers, used by Alembic.
revision = '64fee8e5ab34'

@ -6,8 +6,6 @@ Create Date: 2020-01-18 17:32:21.885068
"""
from alembic import op
import sqlalchemy as sa
from sqlalchemy.dialects import postgresql
from sqlalchemy_searchable import sync_trigger

@ -5,9 +5,8 @@ Revises: df66c78e6791
Create Date: 2020-01-24 21:52:49.744404
"""
from alembic import op
import sqlalchemy as sa
from sqlalchemy.dialects import postgresql
from alembic import op
# revision identifiers, used by Alembic.
revision = '7a48dbd05780'

@ -5,9 +5,8 @@ Revises: dce69ad1e4eb
Create Date: 2019-01-28 20:27:33.760232
"""
from alembic import op
import sqlalchemy as sa
from sqlalchemy.dialects import postgresql
from alembic import op
# revision identifiers, used by Alembic.
revision = '7def3e843d04'

@ -6,8 +6,6 @@ Create Date: 2019-01-29 02:57:50.279918
"""
from alembic import op
import sqlalchemy as sa
from sqlalchemy.dialects import postgresql
# revision identifiers, used by Alembic.
revision = '7ff57806ffd5'

@ -6,8 +6,6 @@ Create Date: 2020-07-12 01:33:19.499459
"""
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision = '838081950f27'

@ -5,9 +5,8 @@ Revises: 7def3e843d04
Create Date: 2019-01-28 20:49:41.831991
"""
from alembic import op
import sqlalchemy as sa
from sqlalchemy.dialects import postgresql
from alembic import op
# revision identifiers, used by Alembic.
revision = '97a9c461bc2d'

@ -5,9 +5,8 @@ Revises: 97a9c461bc2d
Create Date: 2019-01-29 00:37:49.507631
"""
from alembic import op
import sqlalchemy as sa
from sqlalchemy.dialects import postgresql
from alembic import op
# revision identifiers, used by Alembic.
revision = '9ec17b558413'

@ -5,9 +5,8 @@ Revises: 64fee8e5ab34
Create Date: 2020-01-19 19:12:39.402679
"""
from alembic import op
import sqlalchemy as sa
from sqlalchemy.dialects import postgresql
from alembic import op
# revision identifiers, used by Alembic.
revision = 'a0f6c8743362'

@ -6,8 +6,6 @@ Create Date: 2018-05-27 23:51:11.008936
"""
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision = 'b254f55eadd2'

@ -5,11 +5,8 @@ Revises: cb6ab141c522
Create Date: 2020-07-09 00:05:39.845465
"""
from alembic import op
import sqlalchemy as sa
from sqlalchemy import orm, func
from app.models import Package, PackageRelease
from alembic import op
# revision identifiers, used by Alembic.
revision = 'c141a63b2487'

@ -5,11 +5,9 @@ Revises: 7a48dbd05780
Create Date: 2020-07-08 21:03:51.856561
"""
from alembic import op
import sqlalchemy as sa
from alembic import op
from sqlalchemy import orm
from app.models import Package
# revision identifiers, used by Alembic.
revision = 'cb6ab141c522'

@ -5,9 +5,8 @@ Revises: 7ff57806ffd5
Create Date: 2019-07-01 23:27:42.666877
"""
from alembic import op
import sqlalchemy as sa
from sqlalchemy.dialects import postgresql
from alembic import op
# revision identifiers, used by Alembic.
revision = 'd6ae9682c45f'

@ -5,9 +5,8 @@ Revises: a0f6c8743362
Create Date: 2020-01-24 18:39:58.363417
"""
from alembic import op
import sqlalchemy as sa
from sqlalchemy.dialects import postgresql
from alembic import op
# revision identifiers, used by Alembic.
revision = 'df66c78e6791'

@ -5,10 +5,8 @@ Revises: 3a24fc02365e
Create Date: 2020-07-17 23:47:51.096874
"""
from alembic import op
import sqlalchemy as sa
import datetime
from alembic import op
# revision identifiers, used by Alembic.
revision = 'dff4b87e4a76'

@ -6,8 +6,6 @@ Create Date: 2018-05-26 01:55:09.745881
"""
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision = 'ea5a023711e0'

@ -5,9 +5,8 @@ Revises: d6ae9682c45f
Create Date: 2019-11-26 23:43:47.476346
"""
from alembic import op
import sqlalchemy as sa
from sqlalchemy.dialects import postgresql
from alembic import op
# revision identifiers, used by Alembic.
revision = 'fd25bf3e57c3'

@ -15,7 +15,9 @@
# along with this program. If not, see <https://www.gnu.org/licenses/>.
import os, sys, datetime, inspect
import inspect
import os
import sys
if not "FLASK_CONFIG" in os.environ:
os.environ["FLASK_CONFIG"] = "../config.cfg"
@ -29,7 +31,7 @@ currentdir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentfram
parentdir = os.path.dirname(currentdir)
sys.path.insert(0,parentdir)
from app.models import db, User, UserRank
from app.models import db
from app.default_data import populate, populate_test_data
if delete_db and os.path.isfile("db.sqlite"):