Move Github import to backend

Fixes #41

.gitignore

@@ -3,6 +3,8 @@ config.prod.cfg
 *.sqlite
 main.css
 tmp
+log.txt
+*.rdb
 
 # Created by https://www.gitignore.io/api/linux,macos,python,windows
 

app/__init__.py

@@ -12,5 +12,5 @@ menu.Menu(app=app)
 markdown.Markdown(app, extensions=["fenced_code"], safe_mode=True, output_format="html5")
 github = GitHub(app)
 
-from . import models
+from . import models, tasks
 from .views import *

app/static/package_create.js

@@ -1,20 +1,4 @@
 $(function() {
-	function readConfig(text) {
-		var retval = {}
-
-		const lines = text.split("\n")
-		for (var i = 0; i < lines.length; i++) {
-			const idx = lines[i].indexOf("=")
-			if (idx > 0) {
-				const name = lines[i].substring(0, idx - 1).trim()
-				const value = lines[i].substring(idx + 1).trim()
-				retval[name] = value
-			}
-		}
-
-		return retval
-	}
-
 	function finish() {
 		$(".pkg_wiz_1").hide()
 		$(".pkg_wiz_2").hide()
@@ -22,6 +6,49 @@ $(function() {
 		$(".pkg_meta").show()
 	}
 
+	function getJSON(url) {
+		return new Promise(function(resolve, reject) {
+			fetch(url).then(function(response) {
+				response.text().then(function(txt) {
+					resolve(JSON.parse(txt))
+				}).catch(reject)
+			}).catch(reject)
+		})
+	}
+
+	function performTask(url) {
+		return new Promise(function(resolve, reject) {
+			getJSON(url).then(function(startResult) {
+				console.log(startResult)
+				if (typeof startResult.poll_url == "string") {
+					var tries = 0;
+					function retry() {
+						tries++;
+						if (tries > 10) {
+							reject("timeout")
+						} else {
+							console.log("Polling task in " + (tries*100) + "ms")
+							setTimeout(step, tries*100)
+						}
+					}
+					function step() {
+						getJSON(startResult.poll_url).then(function(res) {
+							if (res.status == "SUCCESS") {
+								console.log("Got result")
+								resolve(res.result)
+							} else {
+								retry()
+							}
+						}).catch(retry)
+					}
+					retry()
+				} else {
+					reject("Start task didn't return string!")
+				}
+			}).catch(reject)
+		})
+	}
+
 	function repoIsSupported(url) {
 		try {
 			return URI(url).hostname() == "github.com"
@@ -30,60 +57,6 @@ $(function() {
 		}
 	}
 
-	function getFile(url) {
-		return new Promise(function(resolve, reject) {
-			fetch(url).then(function(response) {
-				response.text().then(resolve).catch(reject)
-			}).catch(reject)
-		})
-	}
-
-	function getInfo(baseUrl) {
-		return new Promise(function(resolve, reject) {
-			getFile(baseUrl + "/mod.conf").then(function(text) {
-				var config = readConfig(text)
-
-				if (config["name"]) {
-					$("#name").val(config["name"])
-				}
-
-				if (config["description"]) {
-					const desc = config["description"]
-					const idx = desc.indexOf(".")
-					$("#shortDesc").val((idx < 5 || idx > 100) ? desc.substring(0, 100) : desc.substring(0, idx))
-					$("#desc").val(desc)
-				}
-
-				resolve()
-			}).catch(function() {
-				reject()
-			})
-		})
-	}
-
-	function importInfo(urlstr) {
-		// Convert to HTTPs
-		try {
-			var url = URI(urlstr).scheme("https")
-				.username("")
-				.password("")
-		} catch(e) {
-			return Promise.reject(e)
-		}
-		// Change domain
-		url = url.hostname("raw.githubusercontent.com")
-
-		// Rewrite path
-		const re = /^\/([^\/]+)\/([^\/]+)\/?$/
-		const results = re.exec(url.path())
-		if (results == null || results.length != 3) {
-			return Promise.reject("Unable to parse URL - please provide a direct URL to the repo")
-		}
-		url.path("/" + results[1] + "/" + results[2].replace(".git", "") + "/master")
-
-		return getInfo(url.toString())
-	}
-
 	$(".pkg_meta").hide()
 	$(".pkg_wiz_1").show()
 	$("#pkg_wiz_1_next").click(function() {
@@ -93,11 +66,22 @@ $(function() {
 			$(".pkg_wiz_2").show()
 			$(".pkg_repo").hide()
 
-			importInfo(repoURL).then(finish).catch(function(x) {
+			performTask("/tasks/getmeta/new/?url=" + encodeURI(repoURL)).then(function(result) {
-				alert(x)
+				console.log(result)
+				$("#name").val(result.name)
+				const desc = result.description || ""
+				if (desc.length > 0) {
+					const idx = desc.indexOf(".")
+					$("#shortDesc").val((idx < 5 || idx > 100) ? desc.substring(0, Math.min(desc.length, 100)) : desc.substring(0, idx))
+					$("#desc").val(desc)
+				}
+				finish()
+			}).catch(function(e) {
+				alert(e)
 				$(".pkg_wiz_1").show()
 				$(".pkg_wiz_2").hide()
 				$(".pkg_repo").show()
+				// finish()
 			})
 		} else {
 			finish()

app/tasks/__init__.py

@@ -0,0 +1,44 @@
+import flask
+from flask.ext.sqlalchemy import SQLAlchemy
+from celery import Celery
+from app import app
+from app.models import *
+
+class FlaskCelery(Celery):
+	def __init__(self, *args, **kwargs):
+		super(FlaskCelery, self).__init__(*args, **kwargs)
+		self.patch_task()
+
+		if 'app' in kwargs:
+			self.init_app(kwargs['app'])
+
+	def patch_task(self):
+		TaskBase = self.Task
+		_celery = self
+
+		class ContextTask(TaskBase):
+			abstract = True
+
+			def __call__(self, *args, **kwargs):
+				if flask.has_app_context():
+					return TaskBase.__call__(self, *args, **kwargs)
+				else:
+					with _celery.app.app_context():
+						return TaskBase.__call__(self, *args, **kwargs)
+
+		self.Task = ContextTask
+
+	def init_app(self, app):
+		self.app = app
+		self.config_from_object(app.config)
+
+def make_celery(app):
+	celery = FlaskCelery(app.import_name, backend=app.config['CELERY_RESULT_BACKEND'],
+					broker=app.config['CELERY_BROKER_URL'])
+
+	celery.init_app(app)
+	return celery
+
+celery = make_celery(app)
+
+from . import importtasks

app/tasks/importtasks.py

@@ -0,0 +1,68 @@
+import flask
+from flask.ext.sqlalchemy import SQLAlchemy
+import urllib.request
+from urllib.parse import urlparse
+
+from app import app
+from app.models import *
+from app.tasks import celery
+
+class GithubURLMaker:
+	def __init__(self, url):
+		# Rewrite path
+		import re
+		m = re.search("^\/([^\/]+)\/([^\/]+)\/?$", url.path)
+		if m is None:
+			return
+
+		user = m.group(1)
+		repo = m.group(2)
+		self.baseUrl = "https://raw.githubusercontent.com/" + user + "/" + repo.replace(".git", "") + "/master"
+
+	def isValid(self):
+		return self.baseUrl is not None
+
+	def getModConfURL(self):
+		return self.baseUrl + "/mod.conf"
+
+def parseConf(string):
+	retval = {}
+	for line in string.split("\n"):
+		idx = line.find("=")
+		if idx > 0:
+			key   = line[:idx-1].strip()
+			value = line[idx+1:].strip()
+			retval[key] = value
+
+	return retval
+
+@celery.task()
+def getMeta(urlstr):
+	url = urlparse(urlstr)
+
+	urlmaker = None
+	if url.netloc == "github.com":
+		urlmaker = GithubURLMaker(url)
+
+	if not urlmaker.isValid():
+		print("Error! Url maker not valid")
+		return
+
+	print(urlmaker.getModConfURL())
+
+	result = {}
+
+	try:
+		contents = urllib.request.urlopen(urlmaker.getModConfURL()).read().decode("utf-8")
+		conf = parseConf(contents)
+		for key in ["name", "description"]:
+			try:
+				result[key] = conf[key]
+			except KeyError:
+				pass
+
+		print(conf)
+	except OSError:
+		print("mod.conf does not exist")
+
+	return result

app/views/__init__.py

@@ -30,4 +30,4 @@ def home_page():
 	packages = Package.query.filter_by(approved=True).all()
 	return render_template("index.html", packages=packages)
 
-from . import users, githublogin, packages, sass
+from . import users, githublogin, packages, sass, api

app/views/api.py

@@ -0,0 +1,35 @@
+from flask import *
+from flask_user import *
+from flask.ext import menu
+from app import app
+from app.models import *
+from app.tasks import celery
+from app.tasks.importtasks import getMeta
+# from celery.result import AsyncResult
+
+from .utils import *
+
+@app.route("/tasks/getmeta/new/")
+def new_getmeta_page():
+	aresult = getMeta.delay(request.args.get("url"))
+	return jsonify({
+		"poll_url": url_for("check_task", id=aresult.id),
+	})
+
+@app.route("/tasks/<id>/")
+def check_task(id):
+    result = celery.AsyncResult(id)
+    status = result.status
+    traceback = result.traceback
+    result = result.result
+    if isinstance(result, Exception):
+        return jsonify({
+            'status': status,
+            'error': str(result),
+            # 'traceback': traceback,
+        })
+    else:
+        return jsonify({
+            'status': status,
+            'result': result,
+        })

log.txt

@@ -1,55 +0,0 @@
-69efdd7 Add user rank changing
-f51224a Add user list
-4898b69 Fix script injection using markdown
-c9073a8 Update README
-287c9e5 Fix suggest changes link
-cd77ad6 Use bash script to start server
-6be5b3e Add registering using Github
-bb9d589 Add EditRequest approval and rejection
-a5042a9 Add EditRequest view page
-dcfd2b0 Add EditRequest creation
-269c8c0 Add packages API
-2a836c1 Add dummy Package.getMainScreenshotURL() function
-73a79d5 Add file upload for releases
-5a9fc51 Add download button and URL
-570dd51 Move package templates to subfolder
-811e830 added me
-8ff5315 Remove access to repos from Github scope
-0385590 Use consistent quotes
-9ddc29e Fix work queue permissions check
-517b8c9 Fix link in README.md
-dc31a98 Add list of packages to profile
-87d7b14 Add packages page to list all types
-4e870bd Add basic search to package list
-8a8b0e5 Improve permission checking in work queue
-7169170 Improve empty text on work queue
-12d58d3 Add more information to approval view
-e5de870 Add work queue
-aed805d Add new package approval
-49a2a91 Add package validation
-a8edae1 Add package creation
-5cc49f2 Add package release editing and approving
-32ac602 Add redirect to full user URL
-bbb46ce Fix incomplete datetime being stored in releases, order releases by date
-d039474 Add create release page
-596f725 Add package releases
-623ca3d Simplify PackageType
-363f9d8 Add not joined rank
-73f24ad Add permission validation to Package.checkPerm()
-bd58f9b Clean up permissions code
-0fae3a6 Fix bug with PackageType to name form
-9fc71a5 User profile: fix typo, add rank
-dad980c Add suggest changes button
-775850b Implement permissions properly
-5a3764f Add edit package
-7c628ca Add example info
-d3484d9 Add more details to package page
-d17535f Fix menu order
-07a9b79 Check type and author in package details
-bc88027 Add package types
-ae60058 Rename mod to package, add README
-358fc4e Add package list and package view
-84f123a Fix profile page
-7d20c49 Add Github login
-7f4faf2 Update dependencies
-366a230 Initial commit

requirements.txt

@@ -6,3 +6,5 @@ Flask-Menu>=0.7.0
 Flask-Markdown>=0.3
 GitHub-Flask>=3.2.0
 pyScss==1.3.4
+celery==4.0.2
+redis==2.10.6