diff --git a/app/models.py b/app/models.py index ece47a9e..d85a8881 100644 --- a/app/models.py +++ b/app/models.py @@ -678,6 +678,25 @@ class EditRequestChange(db.Model): else: setattr(package, self.key.name, self.newValue) + + +class KrockForumTopic(db.Model): + topic_id = db.Column(db.Integer, primary_key=True, autoincrement=False) + author_id = db.Column(db.Integer, db.ForeignKey("user.id"), nullable=False) + author = db.relationship("User") + + ttype = db.Column(db.Integer, nullable=False) + title = db.Column(db.String(200), nullable=False) + name = db.Column(db.String(30), nullable=True) + link = db.Column(db.String(50), nullable=True) + + def getType(self): + if self.ttype == 1 or self.ttype == 2: + return PackageType.MOD + elif self.ttype == 6: + return PackageType.GAME + + # Setup Flask-User db_adapter = SQLAlchemyAdapter(db, User) # Register the User model user_manager = UserManager(db_adapter, app) # Initialize Flask-User diff --git a/app/tasks/forumtasks.py b/app/tasks/forumtasks.py index df8dbb8f..a0339b00 100644 --- a/app/tasks/forumtasks.py +++ b/app/tasks/forumtasks.py @@ -75,3 +75,47 @@ def importUsersFromModList(): db.session.commit() for author in found: checkForumAccount.delay(author, None) + + +BANNED_NAMES = ["mod", "game", "old", "outdated", "wip", "api"] +ALLOWED_TYPES = [1, 2, 6] + +@celery.task() +def importKrocksModList(): + contents = urllib.request.urlopen("http://krock-works.16mb.com/MTstuff/modList.php").read().decode("utf-8") + list = json.loads(contents) + username_to_user = {} + + KrockForumTopic.query.delete() + + for x in list: + type = int(x["type"]) + if not type in ALLOWED_TYPES: + continue + + username = x["author"] + user = username_to_user.get(username) + if user is None: + user = User.query.filter_by(forums_username=username).first() + assert(user is not None) + username_to_user[username] = user + + import re + tags = re.findall("\[([a-z0-9_]+)\]", x["title"]) + name = None + for tag in reversed(tags): + if len(tag) < 50 and not tag in BANNED_NAMES and \ + not re.match("^([a-z][0-9]+)$", tag): + name = tag + break + + topic = KrockForumTopic() + topic.topic_id = x["topicId"] + topic.author_id = user.id + topic.ttype = type + topic.title = x["title"] + topic.name = name + topic.link = x.get("link") + db.session.add(topic) + + db.session.commit() diff --git a/app/templates/admin/list.html b/app/templates/admin/list.html index ff422bad..e5049f90 100644 --- a/app/templates/admin/list.html +++ b/app/templates/admin/list.html @@ -18,6 +18,7 @@ Create users from mod list + Import Krock's mod list Import screenshots from VCS Import dependencies from downloads Set provides to mod name diff --git a/app/templates/todo.html b/app/templates/todo/list.html similarity index 87% rename from app/templates/todo.html rename to app/templates/todo/list.html index b20f26c6..20e8f4fd 100644 --- a/app/templates/todo.html +++ b/app/templates/todo/list.html @@ -51,4 +51,12 @@ {% endfor %} {% endif %} + + Forum Topics without a Package + + + There are + {{ topics_to_add }} packages + to be added to cdb. + {% endblock %} diff --git a/app/templates/todo/topics.html b/app/templates/todo/topics.html new file mode 100644 index 00000000..77f32c07 --- /dev/null +++ b/app/templates/todo/topics.html @@ -0,0 +1,34 @@ +{% extends "base.html" %} + +{% block title %} +Topics to be Added +{% endblock %} + +{% block content %} + Topics to be Added + + + {{ total - (topics | count) }} / {{ total }} packages have been added. + {{ topics | count }} remaining. + + + + + Id + Title + Author + Name + Link + + {% for topic in topics %} + + {{ topic.topic_id }} + [{{ topic.getType().value }}] {{ topic.title }} + {{ topic.author.display_name}} + {{ topic.name or ""}} + {{ topic.link | domain }} + + + {% endfor %} + +{% endblock %} diff --git a/app/templates/users/user_profile_page.html b/app/templates/users/user_profile_page.html index 46312566..7181fc2c 100644 --- a/app/templates/users/user_profile_page.html +++ b/app/templates/users/user_profile_page.html @@ -98,4 +98,27 @@ {% from "macros/packagegridtile.html" import render_pkggrid %} {{ render_pkggrid(packages, show_author=False) }} +{% if topics_to_add %} + + Topics to Add + + + + Id + Title + Name + Link + + {% for topic in topics_to_add %} + + {{ topic.topic_id }} + [{{ topic.getType().value }}] {{ topic.title }} + {{ topic.name or ""}} + {{ topic.link | domain }} + + {% endfor %} + + +{% endif %} + {% endblock %} diff --git a/app/views/admin.py b/app/views/admin.py index fa6639c3..65d52649 100644 --- a/app/views/admin.py +++ b/app/views/admin.py @@ -21,7 +21,7 @@ from flask.ext import menu from app import app from app.models import * from app.tasks.importtasks import importRepoScreenshot, importAllDependencies -from app.tasks.forumtasks import importUsersFromModList +from app.tasks.forumtasks import importUsersFromModList, importKrocksModList from flask_wtf import FlaskForm from wtforms import * from app.utils import loginUser, rank_required @@ -34,6 +34,9 @@ def admin_page(): if action == "importusers": task = importUsersFromModList.delay() return redirect(url_for("check_task", id=task.id, r=url_for("user_list_page"))) + elif action == "importmodlist": + task = importKrocksModList.delay() + return redirect(url_for("check_task", id=task.id, r=url_for("todo_topics_page"))) elif action == "importscreenshots": packages = Package.query \ .filter_by(soft_deleted=False) \ diff --git a/app/views/packages/todo.py b/app/views/packages/todo.py index 26356049..9553ef27 100644 --- a/app/views/packages/todo.py +++ b/app/views/packages/todo.py @@ -40,6 +40,24 @@ def todo_page(): if canApproveScn: screenshots = PackageScreenshot.query.filter_by(approved=False).all() - return render_template("todo.html", title="Reports and Work Queue", + + topics_to_add = KrockForumTopic.query \ + .filter(~ db.exists().where(Package.forums==KrockForumTopic.topic_id)) \ + .count() + + return render_template("todo/list.html", title="Reports and Work Queue", packages=packages, releases=releases, screenshots=screenshots, - canApproveNew=canApproveNew, canApproveRel=canApproveRel, canApproveScn=canApproveScn) + canApproveNew=canApproveNew, canApproveRel=canApproveRel, canApproveScn=canApproveScn, + topics_to_add=topics_to_add) + + +@app.route("/todo/topics/") +@login_required +def todo_topics_page(): + total = KrockForumTopic.query.count() + + topics = KrockForumTopic.query \ + .filter(~ db.exists().where(Package.forums==KrockForumTopic.topic_id)) \ + .all() + + return render_template("todo/topics.html", topics=topics, total=total) diff --git a/app/views/users.py b/app/views/users.py index 84e68285..86219f99 100644 --- a/app/views/users.py +++ b/app/views/users.py @@ -50,12 +50,6 @@ def user_profile_page(username): if not user: abort(404) - packages = user.packages.filter_by(soft_deleted=False) - if not current_user.is_authenticated or (user != current_user and not current_user.canAccessTodoList()): - packages = packages.filter_by(approved=True) - - packages = packages.order_by(db.asc(Package.title)) - form = None if user.checkPerm(current_user, Permission.CHANGE_DNAME) or \ user.checkPerm(current_user, Permission.CHANGE_EMAIL) or \ @@ -97,9 +91,21 @@ def user_profile_page(username): # Redirect to home page return redirect(url_for("user_profile_page", username=username)) + packages = user.packages.filter_by(soft_deleted=False) + if not current_user.is_authenticated or (user != current_user and not current_user.canAccessTodoList()): + packages = packages.filter_by(approved=True) + packages = packages.order_by(db.asc(Package.title)) + + topics_to_add = None + if current_user == user or user.checkPerm(current_user, Permission.CHANGE_AUTHOR): + topics_to_add = KrockForumTopic.query \ + .filter_by(author_id=user.id) \ + .filter(~ db.exists().where(Package.forums==KrockForumTopic.topic_id)) \ + .all() + # Process GET or invalid POST return render_template("users/user_profile_page.html", - user=user, form=form, packages=packages) + user=user, form=form, packages=packages, topics_to_add=topics_to_add) class SetPasswordForm(FlaskForm): email = StringField("Email (Optional)", [Optional(), Email()]) diff --git a/migrations/versions/adad68a5e370_.py b/migrations/versions/adad68a5e370_.py new file mode 100644 index 00000000..97739793 --- /dev/null +++ b/migrations/versions/adad68a5e370_.py @@ -0,0 +1,37 @@ +"""empty message + +Revision ID: adad68a5e370 +Revises: d0bec9e5698e +Create Date: 2018-06-02 18:23:18.123340 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = 'adad68a5e370' +down_revision = 'd0bec9e5698e' +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.create_table('krock_forum_topic', + sa.Column('topic_id', sa.Integer(), autoincrement=False, nullable=False), + sa.Column('author_id', sa.Integer(), nullable=False), + sa.Column('ttype', sa.Integer(), nullable=False), + sa.Column('title', sa.String(length=200), nullable=False), + sa.Column('name', sa.String(length=30), nullable=True), + sa.Column('link', sa.String(length=50), nullable=True), + sa.ForeignKeyConstraint(['author_id'], ['user.id'], ), + sa.PrimaryKeyConstraint('topic_id') + ) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_table('krock_forum_topic') + # ### end Alembic commands ###
+ There are + {{ topics_to_add }} packages + to be added to cdb. +
+ {{ total - (topics | count) }} / {{ total }} packages have been added. + {{ topics | count }} remaining. +