diff --git a/app/tasks/forumtasks.py b/app/tasks/forumtasks.py index e4ddc594..27d90e75 100644 --- a/app/tasks/forumtasks.py +++ b/app/tasks/forumtasks.py @@ -1,9 +1,11 @@ -import flask +import flask, json from flask.ext.sqlalchemy import SQLAlchemy from app import app from app.models import * from app.tasks import celery from .phpbbparser import getProfile +import urllib.request +from urllib.parse import urlparse, quote_plus @celery.task() def checkForumAccount(username, token=None): @@ -24,10 +26,35 @@ def checkForumAccount(username, token=None): # Get github username github_username = profile.get("github") if github_username is not None and github_username.strip() != "": - print("Updated github username") + print("Updated github username for " + user.display_name + " to " + github_username) user.github_username = github_username needsSaving = True # Save if needsSaving: db.session.commit() + +@celery.task() +def importUsersFromModList(): + contents = urllib.request.urlopen("http://krock-works.16mb.com/MTstuff/modList.php").read().decode("utf-8") + list = json.loads(contents) + found = {} + imported = [] + + for user in User.query.all(): + found[user.username] = True + if user.forums_username is not None: + found[user.forums_username] = True + + for x in list: + author = x.get("author") + if author is not None and not author in found: + user = User(author) + user.forums_username = author + imported.append(author) + found[author] = True + db.session.add(user) + + db.session.commit() + for author in found: + checkForumAccount.delay(author, None) diff --git a/app/tasks/phpbbparser.py b/app/tasks/phpbbparser.py index 3932b943..5b3d191d 100644 --- a/app/tasks/phpbbparser.py +++ b/app/tasks/phpbbparser.py @@ -3,7 +3,10 @@ from bs4 import * from urllib.parse import urljoin import urllib.request import os.path -import time +import time, re + +def urlEncodeNonAscii(b): + return re.sub('[\x80-\xFF]', lambda c: '%%%02x' % ord(c.group(0)), b) class Profile: def __init__(self, username): @@ -58,7 +61,7 @@ def __extract_signature(soup): return res[0] def getProfile(url, username): - url = url + "/memberlist.php?mode=viewprofile&un=" + username + url = url + "/memberlist.php?mode=viewprofile&un=" + urlEncodeNonAscii(username) contents = urllib.request.urlopen(url).read().decode("utf-8") soup = BeautifulSoup(contents, "lxml") diff --git a/app/templates/admin/list.html b/app/templates/admin/list.html index 1c0301e0..1db3247f 100644 --- a/app/templates/admin/list.html +++ b/app/templates/admin/list.html @@ -9,4 +9,16 @@