Add git support for importing meta

app/public/static/package_create.js

@@ -10,11 +10,12 @@ $(function() {
 	}
 
 	function repoIsSupported(url) {
-		try {
+		// try {
-			return URI(url).hostname() == "github.com"
+		// 	return URI(url).hostname() == "github.com"
-		} catch(e) {
+		// } catch(e) {
-			return false
+		// 	return false
-		}
+		// }
+		return true
 	}
 
 	$(".pkg_meta").hide()
@@ -36,7 +37,7 @@ $(function() {
 
 			performTask("/tasks/getmeta/new/?url=" + encodeURI(repoURL)).then(function(result) {
 				$("#name").val(result.name || "")
-				setSpecial("#provides_str", result.name || "")
+				setSpecial("#provides_str", result.provides || "")
 				$("#title").val(result.title || "")
 				$("#repo").val(result.repo || repoURL)
 				$("#issueTracker").val(result.issueTracker || "")

app/public/static/polltask.js

@@ -22,7 +22,7 @@ function pollTask(poll_url, disableTimeout) {
 		var tries = 0;
 		function retry() {
 			tries++;
-			if (!disableTimeout && tries > 10) {
+			if (!disableTimeout && tries > 20) {
 				reject("timeout")
 			} else {
 				const interval = Math.min(tries*100, 1000)

app/tasks/importtasks.py

@@ -15,7 +15,7 @@
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
 
-import flask, json, os
+import flask, json, os, git, tempfile
 from flask.ext.sqlalchemy import SQLAlchemy
 from urllib.error import HTTPError
 import urllib.request
@@ -25,51 +25,6 @@ from app.models import *
 from app.tasks import celery, TaskError
 from app.utils import randomString
 
-class GithubURLMaker:
-	def __init__(self, url):
-		# Rewrite path
-		import re
-		m = re.search("^\/([^\/]+)\/([^\/]+)\/?$", url.path)
-		if m is None:
-			return
-
-		user = m.group(1)
-		repo = m.group(2).replace(".git", "")
-		self.baseUrl = "https://raw.githubusercontent.com/{}/{}/master" \
-				.format(user, repo)
-		self.user = user
-		self.repo = repo
-
-	def isValid(self):
-		return self.baseUrl is not None
-
-	def getRepoURL(self):
-		return "https://github.com/{}/{}".format(self.user, self.repo)
-
-	def getIssueTrackerURL(self):
-		return "https://github.com/{}/{}/issues/".format(self.user, self.repo)
-
-	def getModConfURL(self):
-		return self.baseUrl + "/mod.conf"
-
-	def getDescURL(self):
-		return self.baseUrl + "/description.txt"
-
-	def getDependsURL(self):
-		return self.baseUrl + "/depends.txt"
-
-	def getScreenshotURL(self):
-		return self.baseUrl + "/screenshot.png"
-
-	def getCommitsURL(self, branch):
-		return "https://api.github.com/repos/{}/{}/commits?sha={}" \
-				.format(self.user, self.repo, urllib.parse.quote_plus(branch))
-
-	def getCommitDownload(self, commit):
-		return "https://github.com/{}/{}/archive/{}.zip" \
-				.format(self.user, self.repo, commit)
-
-
 krock_list_cache = None
 krock_list_cache_by_name = None
 def getKrockList():
@@ -97,9 +52,9 @@ def getKrockList():
 			return {
 				"title":   x["title"],
 				"author":  x["author"],
-				"name":    x["name"],
+				"name":	x["name"],
 				"topicId": x["topicId"],
-				"link":    x["link"],
+				"link":	x["link"],
 			}
 
 		krock_list_cache = [g(x) for x in list if h(x)]
@@ -143,77 +98,176 @@ def parseConf(string):
 	return retval
 
 
+class PackageTreeNode:
+	def __init__(self, baseDir, author=None, repo=None, name=None):
+		print("Scanning " + baseDir)
+		self.baseDir  = baseDir
+		self.author   = author
+		self.name     = name
+		self.repo     = repo
+		self.meta     = None
+		self.children = []
+
+		# Detect type
+		type = None
+		is_modpack = False
+		if os.path.isfile(baseDir + "/game.conf"):
+			type = PackageType.GAME
+		elif os.path.isfile(baseDir + "/init.lua"):
+			type = PackageType.MOD
+		elif os.path.isfile(baseDir + "/modpack.txt"):
+			type = PackageType.MOD
+			is_modpack = True
+		elif os.path.isdir(baseDir + "/mods"):
+			type = PackageType.GAME
+		elif os.listdir(baseDir) == []:
+			# probably a submodule
+			return
+		else:
+			raise TaskError("Unable to detect package type!")
+
+		self.type = type
+		self.readMetaFiles()
+
+		if self.type == PackageType.GAME:
+			self.addChildrenFromModDir(baseDir + "/mods")
+		elif is_modpack:
+			self.addChildrenFromModDir(baseDir)
+
+
+	def readMetaFiles(self):
+		result = {}
+
+		# .conf file
+		try:
+			with open(self.baseDir + "/mod.conf", "r") as myfile:
+				conf = parseConf(myfile.read())
+				for key in ["name", "description", "title", "depends", "optional_depends"]:
+					try:
+						result[key] = conf[key]
+					except KeyError:
+						pass
+		except IOError:
+			print("description.txt does not exist!")
+
+		# description.txt
+		if not "description" in result:
+			try:
+				with open(self.baseDir + "/description.txt", "r") as myfile:
+					result["description"] = myfile.read()
+			except IOError:
+				print("description.txt does not exist!")
+
+		# depends.txt
+		import re
+		pattern = re.compile("^([a-z0-9_]+)\??$")
+		if not "depends" in result and not "optional_depends" in result:
+			try:
+				with open(self.baseDir + "/depends.txt", "r") as myfile:
+					contents = myfile.read()
+					soft = []
+					hard = []
+					for line in contents.split("\n"):
+						line = line.strip()
+						if pattern.match(line):
+							if line[len(line) - 1] == "?":
+								soft.append( line[:-1])
+							else:
+								hard.append(line)
+
+					result["depends"] = hard
+					result["optional_depends"] = soft
+
+			except IOError:
+				print("depends.txt does not exist!")
+
+		else:
+			if "depends" in result:
+				result["depends"] = [x.strip() for x in result["depends"].split(",")]
+			if "optional_depends" in result:
+				result["optional_depends"] = [x.strip() for x in result["optional_depends"].split(",")]
+
+
+		# Calculate Title
+		if "name" in result and not "title" in result:
+			result["title"] = result["name"].replace("_", " ").title()
+
+		# Calculate short description
+		if "description" in result:
+			desc = result["description"]
+			idx = desc.find(".") + 1
+			cutIdx = min(len(desc), 200 if idx < 5 else idx)
+			result["short_description"] = desc[:cutIdx]
+
+		# Get forum ID
+		info = findModInfo(self.author, result.get("name"), self.repo)
+		if info is not None:
+			result["forumId"] = info.get("topicId")
+
+		if "name" in result:
+			self.name = result["name"]
+			del result["name"]
+
+		self.meta = result
+
+	def addChildrenFromModDir(self, dir):
+		for entry in next(os.walk(dir))[1]:
+			path = dir + "/" + entry
+			if not entry.startswith('.') and os.path.isdir(path):
+				self.children.append(PackageTreeNode(path, name=entry))
+
+
+	def fold(self, attr, key=None, acc=None):
+		if acc is None:
+			acc = set()
+
+		if self.meta is None:
+			return acc
+
+		at = getattr(self, attr)
+		value = at if key is None else at.get(key)
+
+		if isinstance(value, list):
+			acc |= set(value)
+		elif value is not None:
+			acc.add(value)
+
+		for child in self.children:
+			child.fold(attr, key, acc)
+
+		return acc
+
+	def get(self, key):
+		return self.meta.get(key)
+
+
 @celery.task()
 def getMeta(urlstr, author):
 	url = urlparse(urlstr)
 
-	urlmaker = None
+	gitDir = tempfile.gettempdir() + "/" + randomString(10)
-	if url.netloc == "github.com":
+	git.Repo.clone_from(urlstr, gitDir, progress=None, env=None, depth=1)
-		urlmaker = GithubURLMaker(url)
-	else:
-		raise TaskError("Unsupported repo")
 
-	if not urlmaker.isValid():
+	tree = PackageTreeNode(gitDir, author=author, repo=urlstr)
-		raise TaskError("Error! Url maker not valid")
 
 	result = {}
+	result["name"] = tree.name
+	result["provides"] = tree.fold("name")
+	result["type"] = tree.type.name
 
-	result["repo"] = urlmaker.getRepoURL()
+	for key in ["depends", "optional_depends"]:
-	result["issueTracker"] = urlmaker.getIssueTrackerURL()
+		result[key] = tree.fold("meta", key)
 
-	try:
+	for key in ["title", "repo", "issueTracker", "forumId", "description", "short_description"]:
-		contents = urllib.request.urlopen(urlmaker.getModConfURL()).read().decode("utf-8")
+		result[key] = tree.get(key)
-		conf = parseConf(contents)
-		for key in ["name", "description", "title", "depends", "optional_depends"]:
-			try:
-				result[key] = conf[key]
-			except KeyError:
-				pass
-	except HTTPError:
-		print("mod.conf does not exist")
 
-	if "name" in result:
+	for mod in result["provides"]:
-		result["title"] = result["name"].replace("_", " ").title()
+		result["depends"].discard(mod)
+		result["optional_depends"].discard(mod)
 
-	if not "description" in result:
+	for key, value in result.items():
-		try:
+		if isinstance(value, set):
-			contents = urllib.request.urlopen(urlmaker.getDescURL()).read().decode("utf-8")
+			result[key] = list(value)
-			result["description"] = contents.strip()
-		except HTTPError:
-			print("description.txt does not exist!")
-
-	import re
-	pattern = re.compile("^([a-z0-9_]+)\??$")
-	if not "depends" in result and not "optional_depends" in result:
-		try:
-			contents = urllib.request.urlopen(urlmaker.getDependsURL()).read().decode("utf-8")
-			soft = []
-			hard = []
-			for line in contents.split("\n"):
-				line = line.strip()
-				if pattern.match(line):
-					if line[len(line) - 1] == "?":
-						soft.append( line[:-1])
-					else:
-						hard.append(line)
-
-			result["depends"] = ",".join(hard)
-			result["optional_depends"] = ",".join(soft)
-
-
-		except HTTPError:
-			print("depends.txt does not exist!")
-
-	if "description" in result:
-		desc = result["description"]
-		idx = desc.find(".") + 1
-		cutIdx = min(len(desc), 200 if idx < 5 else idx)
-		result["short_description"] = desc[:cutIdx]
-
-
-	info = findModInfo(author, result.get("name"), result["repo"])
-	if info is not None:
-		result["forumId"] = info.get("topicId")
 
 	return result
 
@@ -281,7 +335,7 @@ def importRepoScreenshot(id):
 		ss.approved = True
 		ss.package = package
 		ss.title   = "screenshot.png"
-		ss.url     = "/uploads/" + filename
+		ss.url	 = "/uploads/" + filename
 		db.session.add(ss)
 		db.session.commit()
 

app/templates/packages/create_edit.html

@@ -49,7 +49,7 @@
 
 		<div class="pkg_wiz_1">
 			<p>Enter the repo URL for the package.
-			If it's hosted on Github then metadata will automatically be imported.</p>
+			If the repo uses git then the metadata will be automatically imported.</p>
 
 			<p>Leave blank if you don't have a repo.</p>
 		</div>
@@ -61,7 +61,7 @@
 		</div>
 
 		<div class="pkg_wiz_2">
-			Importing...
+			Importing... (This may take a while)
 		</div>
 
 		{{ render_field(form.website, class_="pkg_meta") }}

requirements.txt

@@ -13,3 +13,4 @@ lxml==4.2.1
 Flask-FlatPages==0.6
 Flask-Migrate==2.1.1
 pillow==5.1.0
+GitPython==2.1.10