Add topics todo list based on forum parser

app/models.py

@@ -678,6 +678,25 @@ class EditRequestChange(db.Model):
 		else:
 			setattr(package, self.key.name, self.newValue)
 
+
+
+class KrockForumTopic(db.Model):
+	topic_id  = db.Column(db.Integer, primary_key=True, autoincrement=False)
+	author_id = db.Column(db.Integer, db.ForeignKey("user.id"), nullable=False)
+	author    = db.relationship("User")
+
+	ttype     = db.Column(db.Integer, nullable=False)
+	title     = db.Column(db.String(200), nullable=False)
+	name      = db.Column(db.String(30), nullable=True)
+	link      = db.Column(db.String(50), nullable=True)
+
+	def getType(self):
+		if self.ttype == 1 or self.ttype == 2:
+			return PackageType.MOD
+		elif self.ttype == 6:
+			return PackageType.GAME
+
+
 # Setup Flask-User
 db_adapter = SQLAlchemyAdapter(db, User)        # Register the User model
 user_manager = UserManager(db_adapter, app)     # Initialize Flask-User

app/tasks/forumtasks.py

@@ -75,3 +75,47 @@ def importUsersFromModList():
 	db.session.commit()
 	for author in found:
 		checkForumAccount.delay(author, None)
+
+
+BANNED_NAMES = ["mod", "game", "old", "outdated", "wip", "api"]
+ALLOWED_TYPES = [1, 2, 6]
+
+@celery.task()
+def importKrocksModList():
+	contents = urllib.request.urlopen("http://krock-works.16mb.com/MTstuff/modList.php").read().decode("utf-8")
+	list = json.loads(contents)
+	username_to_user = {}
+
+	KrockForumTopic.query.delete()
+
+	for x in list:
+		type = int(x["type"])
+		if not type in ALLOWED_TYPES:
+			continue
+
+		username = x["author"]
+		user = username_to_user.get(username)
+		if user is None:
+			user = User.query.filter_by(forums_username=username).first()
+			assert(user is not None)
+			username_to_user[username] = user
+
+		import re
+		tags = re.findall("\[([a-z0-9_]+)\]", x["title"])
+		name = None
+		for tag in reversed(tags):
+			if len(tag) < 50 and not tag in BANNED_NAMES and \
+					not re.match("^([a-z][0-9]+)$", tag):
+				name = tag
+				break
+
+		topic = KrockForumTopic()
+		topic.topic_id  = x["topicId"]
+		topic.author_id = user.id
+		topic.ttype     = type
+		topic.title     = x["title"]
+		topic.name      = name
+		topic.link      = x.get("link")
+		db.session.add(topic)
+
+	db.session.commit()

app/templates/admin/list.html

@@ -18,6 +18,7 @@
 			<input type="hidden" name="csrf_token" value="{{ csrf_token() }}" />
 			<select name="action">
 				<option value="importusers">Create users from mod list</option>
+				<option value="importmodlist">Import Krock's mod list</option>
 				<option value="importscreenshots" selected>Import screenshots from VCS</option>
 				<option value="importdepends">Import dependencies from downloads</option>
 				<option value="modprovides">Set provides to mod name</option>

app/templates/todo/list.html

@@ -51,4 +51,12 @@
 			{% endfor %}
 		</ul>
 	{% endif %}
+
+	<h2>Forum Topics without a Package</h2>
+
+	<p>
+		There are
+		<a href="{{ url_for('todo_topics_page') }}">{{ topics_to_add }} packages</a>
+		to be added to cdb.
+	</p>
 {% endblock %}

app/templates/todo/topics.html

@@ -0,0 +1,34 @@
+{% extends "base.html" %}
+
+{% block title %}
+Topics to be Added
+{% endblock %}
+
+{% block content %}
+	<h1>Topics to be Added</h1>
+
+	<p>
+		{{ total - (topics | count) }} / {{ total }} packages have been added.
+		{{ topics | count }} remaining.
+	</p>
+
+	<table>
+		<tr>
+			<th>Id</th>
+			<th>Title</th>
+			<th>Author</th>
+			<th>Name</th>
+			<th>Link</th>
+		</tr>
+		{% for topic in topics %}
+			<tr>
+				<td>{{ topic.topic_id }}</td>
+				<td>[{{ topic.getType().value }}] <a href="https://forum.minetest.net/viewtopic.php?t={{ topic.topic_id}}">{{ topic.title }}</a></td>
+				<td><a href="{{ url_for('user_profile_page', username=topic.author.username) }}">{{ topic.author.display_name}}</a></td>
+				<td>{{ topic.name or ""}}</td>
+				<td><a href="{{ topic.link }}">{{ topic.link | domain }}</a></td>
+
+			</tr>
+		{% endfor %}
+	</table>
+{% endblock %}

app/templates/users/user_profile_page.html

@@ -98,4 +98,27 @@
 {% from "macros/packagegridtile.html" import render_pkggrid %}
 {{ render_pkggrid(packages, show_author=False) }}
 
+{% if topics_to_add %}
+	<div class="box box_grey">
+		<h2>Topics to Add</h2>
+
+		<table class="box-body">
+			<tr>
+				<th>Id</th>
+				<th>Title</th>
+				<th>Name</th>
+				<th>Link</th>
+			</tr>
+			{% for topic in topics_to_add %}
+				<tr>
+					<td>{{ topic.topic_id }}</td>
+					<td>[{{ topic.getType().value }}] <a href="https://forum.minetest.net/viewtopic.php?t={{ topic.topic_id}}">{{ topic.title }}</a></td>
+					<td>{{ topic.name or ""}}</td>
+					<td><a href="{{ topic.link }}">{{ topic.link | domain }}</a></td>
+				</tr>
+			{% endfor %}
+		</table>
+	</div>
+{% endif %}
+
 {% endblock %}

app/views/admin.py

@@ -21,7 +21,7 @@ from flask.ext import menu
 from app import app
 from app.models import *
 from app.tasks.importtasks import importRepoScreenshot, importAllDependencies
-from app.tasks.forumtasks  import importUsersFromModList
+from app.tasks.forumtasks  import importUsersFromModList, importKrocksModList
 from flask_wtf import FlaskForm
 from wtforms import *
 from app.utils import loginUser, rank_required
@@ -34,6 +34,9 @@ def admin_page():
 		if action == "importusers":
 			task = importUsersFromModList.delay()
 			return redirect(url_for("check_task", id=task.id, r=url_for("user_list_page")))
+		elif action == "importmodlist":
+			task = importKrocksModList.delay()
+			return redirect(url_for("check_task", id=task.id, r=url_for("todo_topics_page")))
 		elif action == "importscreenshots":
 			packages = Package.query \
 				.filter_by(soft_deleted=False) \

app/views/packages/todo.py

@@ -40,6 +40,24 @@ def todo_page():
 	if canApproveScn:
 		screenshots = PackageScreenshot.query.filter_by(approved=False).all()
 
-	return render_template("todo.html", title="Reports and Work Queue",
+
+	topics_to_add = KrockForumTopic.query \
+			.filter(~ db.exists().where(Package.forums==KrockForumTopic.topic_id)) \
+			.count()
+
+	return render_template("todo/list.html", title="Reports and Work Queue",
 		packages=packages, releases=releases, screenshots=screenshots,
-		canApproveNew=canApproveNew, canApproveRel=canApproveRel, canApproveScn=canApproveScn)
+		canApproveNew=canApproveNew, canApproveRel=canApproveRel, canApproveScn=canApproveScn,
+		topics_to_add=topics_to_add)
+
+
+@app.route("/todo/topics/")
+@login_required
+def todo_topics_page():
+	total = KrockForumTopic.query.count()
+
+	topics = KrockForumTopic.query \
+			.filter(~ db.exists().where(Package.forums==KrockForumTopic.topic_id)) \
+			.all()
+
+	return render_template("todo/topics.html", topics=topics, total=total)

app/views/users.py

@@ -50,12 +50,6 @@ def user_profile_page(username):
 	if not user:
 		abort(404)
 
-	packages = user.packages.filter_by(soft_deleted=False)
-	if not current_user.is_authenticated or (user != current_user and not current_user.canAccessTodoList()):
-		packages = packages.filter_by(approved=True)
-
-	packages = packages.order_by(db.asc(Package.title))
-
 	form = None
 	if user.checkPerm(current_user, Permission.CHANGE_DNAME) or \
 			user.checkPerm(current_user, Permission.CHANGE_EMAIL) or \
@@ -97,9 +91,21 @@ def user_profile_page(username):
 			# Redirect to home page
 			return redirect(url_for("user_profile_page", username=username))
 
+	packages = user.packages.filter_by(soft_deleted=False)
+	if not current_user.is_authenticated or (user != current_user and not current_user.canAccessTodoList()):
+		packages = packages.filter_by(approved=True)
+	packages = packages.order_by(db.asc(Package.title))
+
+	topics_to_add = None
+	if current_user == user or user.checkPerm(current_user, Permission.CHANGE_AUTHOR):
+		topics_to_add = KrockForumTopic.query \
+					.filter_by(author_id=user.id) \
+					.filter(~ db.exists().where(Package.forums==KrockForumTopic.topic_id)) \
+					.all()
+
 	# Process GET or invalid POST
 	return render_template("users/user_profile_page.html",
-			user=user, form=form, packages=packages)
+			user=user, form=form, packages=packages, topics_to_add=topics_to_add)
 
 class SetPasswordForm(FlaskForm):
 	email = StringField("Email (Optional)", [Optional(), Email()])

migrations/versions/adad68a5e370_.py

@@ -0,0 +1,37 @@
+"""empty message
+
+Revision ID: adad68a5e370
+Revises: d0bec9e5698e
+Create Date: 2018-06-02 18:23:18.123340
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = 'adad68a5e370'
+down_revision = 'd0bec9e5698e'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table('krock_forum_topic',
+    sa.Column('topic_id', sa.Integer(), autoincrement=False, nullable=False),
+    sa.Column('author_id', sa.Integer(), nullable=False),
+    sa.Column('ttype', sa.Integer(), nullable=False),
+    sa.Column('title', sa.String(length=200), nullable=False),
+    sa.Column('name', sa.String(length=30), nullable=True),
+    sa.Column('link', sa.String(length=50), nullable=True),
+    sa.ForeignKeyConstraint(['author_id'], ['user.id'], ),
+    sa.PrimaryKeyConstraint('topic_id')
+    )
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_table('krock_forum_topic')
+    # ### end Alembic commands ###