Add topics todo list based on forum parser

This commit is contained in:
rubenwardy 2018-06-02 18:22:57 +01:00
parent 8984adaa72
commit 70afb94d3b
No known key found for this signature in database
GPG Key ID: A1E29D52FF81513C
10 changed files with 203 additions and 10 deletions

@ -678,6 +678,25 @@ class EditRequestChange(db.Model):
else:
setattr(package, self.key.name, self.newValue)
class KrockForumTopic(db.Model):
topic_id = db.Column(db.Integer, primary_key=True, autoincrement=False)
author_id = db.Column(db.Integer, db.ForeignKey("user.id"), nullable=False)
author = db.relationship("User")
ttype = db.Column(db.Integer, nullable=False)
title = db.Column(db.String(200), nullable=False)
name = db.Column(db.String(30), nullable=True)
link = db.Column(db.String(50), nullable=True)
def getType(self):
if self.ttype == 1 or self.ttype == 2:
return PackageType.MOD
elif self.ttype == 6:
return PackageType.GAME
# Setup Flask-User
db_adapter = SQLAlchemyAdapter(db, User) # Register the User model
user_manager = UserManager(db_adapter, app) # Initialize Flask-User

@ -75,3 +75,47 @@ def importUsersFromModList():
db.session.commit()
for author in found:
checkForumAccount.delay(author, None)
BANNED_NAMES = ["mod", "game", "old", "outdated", "wip", "api"]
ALLOWED_TYPES = [1, 2, 6]
@celery.task()
def importKrocksModList():
contents = urllib.request.urlopen("http://krock-works.16mb.com/MTstuff/modList.php").read().decode("utf-8")
list = json.loads(contents)
username_to_user = {}
KrockForumTopic.query.delete()
for x in list:
type = int(x["type"])
if not type in ALLOWED_TYPES:
continue
username = x["author"]
user = username_to_user.get(username)
if user is None:
user = User.query.filter_by(forums_username=username).first()
assert(user is not None)
username_to_user[username] = user
import re
tags = re.findall("\[([a-z0-9_]+)\]", x["title"])
name = None
for tag in reversed(tags):
if len(tag) < 50 and not tag in BANNED_NAMES and \
not re.match("^([a-z][0-9]+)$", tag):
name = tag
break
topic = KrockForumTopic()
topic.topic_id = x["topicId"]
topic.author_id = user.id
topic.ttype = type
topic.title = x["title"]
topic.name = name
topic.link = x.get("link")
db.session.add(topic)
db.session.commit()

@ -18,6 +18,7 @@
<input type="hidden" name="csrf_token" value="{{ csrf_token() }}" />
<select name="action">
<option value="importusers">Create users from mod list</option>
<option value="importmodlist">Import Krock's mod list</option>
<option value="importscreenshots" selected>Import screenshots from VCS</option>
<option value="importdepends">Import dependencies from downloads</option>
<option value="modprovides">Set provides to mod name</option>

@ -51,4 +51,12 @@
{% endfor %}
</ul>
{% endif %}
<h2>Forum Topics without a Package</h2>
<p>
There are
<a href="{{ url_for('todo_topics_page') }}">{{ topics_to_add }} packages</a>
to be added to cdb.
</p>
{% endblock %}

@ -0,0 +1,34 @@
{% extends "base.html" %}
{% block title %}
Topics to be Added
{% endblock %}
{% block content %}
<h1>Topics to be Added</h1>
<p>
{{ total - (topics | count) }} / {{ total }} packages have been added.
{{ topics | count }} remaining.
</p>
<table>
<tr>
<th>Id</th>
<th>Title</th>
<th>Author</th>
<th>Name</th>
<th>Link</th>
</tr>
{% for topic in topics %}
<tr>
<td>{{ topic.topic_id }}</td>
<td>[{{ topic.getType().value }}] <a href="https://forum.minetest.net/viewtopic.php?t={{ topic.topic_id}}">{{ topic.title }}</a></td>
<td><a href="{{ url_for('user_profile_page', username=topic.author.username) }}">{{ topic.author.display_name}}</a></td>
<td>{{ topic.name or ""}}</td>
<td><a href="{{ topic.link }}">{{ topic.link | domain }}</a></td>
</tr>
{% endfor %}
</table>
{% endblock %}

@ -98,4 +98,27 @@
{% from "macros/packagegridtile.html" import render_pkggrid %}
{{ render_pkggrid(packages, show_author=False) }}
{% if topics_to_add %}
<div class="box box_grey">
<h2>Topics to Add</h2>
<table class="box-body">
<tr>
<th>Id</th>
<th>Title</th>
<th>Name</th>
<th>Link</th>
</tr>
{% for topic in topics_to_add %}
<tr>
<td>{{ topic.topic_id }}</td>
<td>[{{ topic.getType().value }}] <a href="https://forum.minetest.net/viewtopic.php?t={{ topic.topic_id}}">{{ topic.title }}</a></td>
<td>{{ topic.name or ""}}</td>
<td><a href="{{ topic.link }}">{{ topic.link | domain }}</a></td>
</tr>
{% endfor %}
</table>
</div>
{% endif %}
{% endblock %}

@ -21,7 +21,7 @@ from flask.ext import menu
from app import app
from app.models import *
from app.tasks.importtasks import importRepoScreenshot, importAllDependencies
from app.tasks.forumtasks import importUsersFromModList
from app.tasks.forumtasks import importUsersFromModList, importKrocksModList
from flask_wtf import FlaskForm
from wtforms import *
from app.utils import loginUser, rank_required
@ -34,6 +34,9 @@ def admin_page():
if action == "importusers":
task = importUsersFromModList.delay()
return redirect(url_for("check_task", id=task.id, r=url_for("user_list_page")))
elif action == "importmodlist":
task = importKrocksModList.delay()
return redirect(url_for("check_task", id=task.id, r=url_for("todo_topics_page")))
elif action == "importscreenshots":
packages = Package.query \
.filter_by(soft_deleted=False) \

@ -40,6 +40,24 @@ def todo_page():
if canApproveScn:
screenshots = PackageScreenshot.query.filter_by(approved=False).all()
return render_template("todo.html", title="Reports and Work Queue",
topics_to_add = KrockForumTopic.query \
.filter(~ db.exists().where(Package.forums==KrockForumTopic.topic_id)) \
.count()
return render_template("todo/list.html", title="Reports and Work Queue",
packages=packages, releases=releases, screenshots=screenshots,
canApproveNew=canApproveNew, canApproveRel=canApproveRel, canApproveScn=canApproveScn)
canApproveNew=canApproveNew, canApproveRel=canApproveRel, canApproveScn=canApproveScn,
topics_to_add=topics_to_add)
@app.route("/todo/topics/")
@login_required
def todo_topics_page():
total = KrockForumTopic.query.count()
topics = KrockForumTopic.query \
.filter(~ db.exists().where(Package.forums==KrockForumTopic.topic_id)) \
.all()
return render_template("todo/topics.html", topics=topics, total=total)

@ -50,12 +50,6 @@ def user_profile_page(username):
if not user:
abort(404)
packages = user.packages.filter_by(soft_deleted=False)
if not current_user.is_authenticated or (user != current_user and not current_user.canAccessTodoList()):
packages = packages.filter_by(approved=True)
packages = packages.order_by(db.asc(Package.title))
form = None
if user.checkPerm(current_user, Permission.CHANGE_DNAME) or \
user.checkPerm(current_user, Permission.CHANGE_EMAIL) or \
@ -97,9 +91,21 @@ def user_profile_page(username):
# Redirect to home page
return redirect(url_for("user_profile_page", username=username))
packages = user.packages.filter_by(soft_deleted=False)
if not current_user.is_authenticated or (user != current_user and not current_user.canAccessTodoList()):
packages = packages.filter_by(approved=True)
packages = packages.order_by(db.asc(Package.title))
topics_to_add = None
if current_user == user or user.checkPerm(current_user, Permission.CHANGE_AUTHOR):
topics_to_add = KrockForumTopic.query \
.filter_by(author_id=user.id) \
.filter(~ db.exists().where(Package.forums==KrockForumTopic.topic_id)) \
.all()
# Process GET or invalid POST
return render_template("users/user_profile_page.html",
user=user, form=form, packages=packages)
user=user, form=form, packages=packages, topics_to_add=topics_to_add)
class SetPasswordForm(FlaskForm):
email = StringField("Email (Optional)", [Optional(), Email()])

@ -0,0 +1,37 @@
"""empty message
Revision ID: adad68a5e370
Revises: d0bec9e5698e
Create Date: 2018-06-02 18:23:18.123340
"""
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision = 'adad68a5e370'
down_revision = 'd0bec9e5698e'
branch_labels = None
depends_on = None
def upgrade():
# ### commands auto generated by Alembic - please adjust! ###
op.create_table('krock_forum_topic',
sa.Column('topic_id', sa.Integer(), autoincrement=False, nullable=False),
sa.Column('author_id', sa.Integer(), nullable=False),
sa.Column('ttype', sa.Integer(), nullable=False),
sa.Column('title', sa.String(length=200), nullable=False),
sa.Column('name', sa.String(length=30), nullable=True),
sa.Column('link', sa.String(length=50), nullable=True),
sa.ForeignKeyConstraint(['author_id'], ['user.id'], ),
sa.PrimaryKeyConstraint('topic_id')
)
# ### end Alembic commands ###
def downgrade():
# ### commands auto generated by Alembic - please adjust! ###
op.drop_table('krock_forum_topic')
# ### end Alembic commands ###