Add import users from Krock's mod list feature

app/tasks/forumtasks.py

@@ -1,9 +1,11 @@
-import flask
+import flask, json
 from flask.ext.sqlalchemy import SQLAlchemy
 from app import app
 from app.models import *
 from app.tasks import celery
 from .phpbbparser import getProfile
+import urllib.request
+from urllib.parse import urlparse, quote_plus
 
 @celery.task()
 def checkForumAccount(username, token=None):
@@ -24,10 +26,35 @@ def checkForumAccount(username, token=None):
 	# Get github username
 	github_username = profile.get("github")
 	if github_username is not None and github_username.strip() != "":
-		print("Updated github username")
+		print("Updated github username for " + user.display_name + " to " + github_username)
 		user.github_username = github_username
 		needsSaving = True
 
 	# Save
 	if needsSaving:
 		db.session.commit()
+
+@celery.task()
+def importUsersFromModList():
+	contents = urllib.request.urlopen("http://krock-works.16mb.com/MTstuff/modList.php").read().decode("utf-8")
+	list = json.loads(contents)
+	found = {}
+	imported = []
+
+	for user in User.query.all():
+		found[user.username] = True
+		if user.forums_username is not None:
+			found[user.forums_username] = True
+
+	for x in list:
+		author = x.get("author")
+		if author is not None and not author in found:
+			user = User(author)
+			user.forums_username = author
+			imported.append(author)
+			found[author] = True
+			db.session.add(user)
+
+	db.session.commit()
+	for author in found:
+		checkForumAccount.delay(author, None)

app/tasks/phpbbparser.py

@@ -3,7 +3,10 @@ from bs4 import *
 from urllib.parse import urljoin
 import urllib.request
 import os.path
-import time
+import time, re
+
+def urlEncodeNonAscii(b):
+	return re.sub('[\x80-\xFF]', lambda c: '%%%02x' % ord(c.group(0)), b)
 
 class Profile:
 	def __init__(self, username):
@@ -58,7 +61,7 @@ def __extract_signature(soup):
 		return res[0]
 
 def getProfile(url, username):
-	url = url + "/memberlist.php?mode=viewprofile&un=" + username
+	url = url + "/memberlist.php?mode=viewprofile&un=" + urlEncodeNonAscii(username)
 
 	contents = urllib.request.urlopen(url).read().decode("utf-8")
 	soup = BeautifulSoup(contents, "lxml")

app/templates/admin/list.html

@@ -9,4 +9,16 @@
 		<li><a href="{{ url_for('user_list_page') }}">User list</a></li>
 		<li><a href="{{ url_for('switch_user_page') }}">Sign in as another user</a></li>
 	</ul>
+
+	<div class="box box_grey">
+		<h2>Do action</h2>
+
+		<form method="post" action="">
+			<input type="hidden" name="csrf_token" value="{{ csrf_token() }}" />
+			<select name="action">
+				<option value="importusers" selected>Create users from mod list</option>
+			</select>
+			<input type="submit" value="Start" />
+		</form>
+	</div>
 {% endblock %}

app/templates/admin/switch_user_page.html

@@ -5,6 +5,8 @@
 {% endblock %}
 
 {% block content %}
+	<h2>Log in as another user</h2>
+
 	{% from "macros/forms.html" import render_field, render_submit_field %}
 	<form method="POST" action="">
 		{{ form.hidden_tag() }}

app/views/admin.py

@@ -3,15 +3,24 @@ from flask_user import *
 from flask.ext import menu
 from app import app
 from app.models import *
+from app.tasks.forumtasks import importUsersFromModList
 from flask_wtf import FlaskForm
 from wtforms import *
 from .utils import loginUser, rank_required
 
 @menu.register_menu(app, ".admin", "Admin", order=30,
 		visible_when=lambda: current_user.rank.atLeast(UserRank.ADMIN))
-@app.route("/admin/")
+@app.route("/admin/", methods=["GET", "POST"])
 @rank_required(UserRank.ADMIN)
 def admin_page():
+	if request.method == "POST":
+		action = request.form["action"]
+		if action == "importusers":
+			task = importUsersFromModList.delay()
+			return redirect(url_for("check_task", id=task.id, r=url_for("user_list_page")))
+		else:
+			flash("Unknown action: " + action, "error")
+
 	return render_template("admin/list.html")
 
 class SwitchUserForm(FlaskForm):