From 3992b19be321116ad425d88932e49164c61c2738 Mon Sep 17 00:00:00 2001 From: rubenwardy Date: Tue, 21 Apr 2020 20:35:05 +0100 Subject: [PATCH] Optimise SQL queries --- app/blueprints/api/endpoints.py | 3 +++ app/blueprints/homepage/__init__.py | 14 ++++++++++---- app/blueprints/packages/packages.py | 6 ++++++ app/models.py | 11 ++++++++--- config.example.cfg | 3 +++ 5 files changed, 30 insertions(+), 7 deletions(-) diff --git a/app/blueprints/api/endpoints.py b/app/blueprints/api/endpoints.py index 7c7a44a7..65af3b08 100644 --- a/app/blueprints/api/endpoints.py +++ b/app/blueprints/api/endpoints.py @@ -28,6 +28,9 @@ from app.querybuilder import QueryBuilder @bp.route("/api/packages/") def packages(): + import sys + print("\n\n############", file=sys.stderr) + qb = QueryBuilder(request.args) query = qb.buildPackageQuery() ver = qb.getMinetestVersion() diff --git a/app/blueprints/homepage/__init__.py b/app/blueprints/homepage/__init__.py index df638df7..5df612f7 100644 --- a/app/blueprints/homepage/__init__.py +++ b/app/blueprints/homepage/__init__.py @@ -4,17 +4,23 @@ bp = Blueprint("homepage", __name__) from app.models import * import flask_menu as menu +from sqlalchemy.orm import joinedload from sqlalchemy.sql.expression import func @bp.route("/") @menu.register_menu(bp, ".", "Home") def home(): + def join(query): + return query.options( \ + joinedload(Package.license), \ + joinedload(Package.media_license)) + query = Package.query.filter_by(approved=True, soft_deleted=False) count = query.count() - new = query.order_by(db.desc(Package.created_at)).limit(8).all() - pop_mod = query.filter_by(type=PackageType.MOD).order_by(db.desc(Package.score)).limit(8).all() - pop_gam = query.filter_by(type=PackageType.GAME).order_by(db.desc(Package.score)).limit(4).all() - pop_txp = query.filter_by(type=PackageType.TXP).order_by(db.desc(Package.score)).limit(4).all() + new = join(query.order_by(db.desc(Package.created_at))).limit(8).all() + pop_mod = join(query.filter_by(type=PackageType.MOD).order_by(db.desc(Package.score))).limit(8).all() + pop_gam = join(query.filter_by(type=PackageType.GAME).order_by(db.desc(Package.score))).limit(4).all() + pop_txp = join(query.filter_by(type=PackageType.TXP).order_by(db.desc(Package.score))).limit(4).all() downloads_result = db.session.query(func.sum(PackageRelease.downloads)).one_or_none() downloads = 0 if not downloads_result or not downloads_result[0] else downloads_result[0] return render_template("index.html", count=count, downloads=downloads, \ diff --git a/app/blueprints/packages/packages.py b/app/blueprints/packages/packages.py index a7d5793c..700fc44c 100644 --- a/app/blueprints/packages/packages.py +++ b/app/blueprints/packages/packages.py @@ -31,6 +31,7 @@ from wtforms import * from wtforms.validators import * from wtforms.ext.sqlalchemy.fields import QuerySelectField, QuerySelectMultipleField from sqlalchemy import or_, func +from sqlalchemy.orm import joinedload, subqueryload @menu.register_menu(bp, ".mods", "Mods", order=11, endpoint_arguments_constructor=lambda: { 'type': 'mod' }) @@ -43,6 +44,11 @@ def list_all(): query = qb.buildPackageQuery() title = qb.title + query = query.options( \ + joinedload(Package.license), \ + joinedload(Package.media_license), \ + subqueryload(Package.tags)) + if qb.lucky: package = query.first() if package: diff --git a/app/models.py b/app/models.py index 4eed0c22..58241894 100644 --- a/app/models.py +++ b/app/models.py @@ -147,7 +147,7 @@ class User(db.Model, UserMixin): notifications = db.relationship("Notification", primaryjoin="User.id==Notification.user_id") # causednotifs = db.relationship("Notification", backref="causer", lazy="dynamic") - packages = db.relationship("Package", backref="author", lazy="dynamic") + packages = db.relationship("Package", backref=db.backref("author", lazy="joined"), lazy="dynamic") requests = db.relationship("EditRequest", backref="author", lazy="dynamic") threads = db.relationship("Thread", backref="author", lazy="dynamic") tokens = db.relationship("APIToken", backref="owner", lazy="dynamic") @@ -437,12 +437,12 @@ class Package(db.Model): forums = db.Column(db.Integer, nullable=True) provides = db.relationship("MetaPackage", \ - secondary=provides, lazy="subquery", order_by=db.asc("name"), \ + secondary=provides, lazy="select", order_by=db.asc("name"), \ backref=db.backref("packages", lazy="dynamic", order_by=db.desc("score"))) dependencies = db.relationship("Dependency", backref="depender", lazy="dynamic", foreign_keys=[Dependency.depender_id]) - tags = db.relationship("Tag", secondary=tags, lazy="subquery", + tags = db.relationship("Tag", secondary=tags, lazy="select", backref=db.backref("packages", lazy=True)) releases = db.relationship("PackageRelease", backref="package", @@ -1135,3 +1135,8 @@ class ForumTopic(db.Model): # Setup Flask-User user_manager = UserManager(app, db, User) + +if app.config.get("LOG_SQL"): + import logging + logging.basicConfig() + logging.getLogger('sqlalchemy.engine').setLevel(logging.INFO) diff --git a/config.example.cfg b/config.example.cfg index 7f2a2981..25fe9e89 100644 --- a/config.example.cfg +++ b/config.example.cfg @@ -32,6 +32,9 @@ MAIL_UTILS_ERROR_SEND_TO = [""] UPLOAD_DIR = "/var/cdb/uploads/" THUMBNAIL_DIR = "/var/cdb/thumbnails/" +TEMPLATES_AUTO_RELOAD = False +LOG_SQL = False + LANGUAGES = { 'en': 'English', }