Add support for zip uploads in the API

Fixes #261

app/blueprints/api/endpoints.py

@@ -19,7 +19,7 @@ from flask import *
 from flask_login import current_user, login_required
 from . import bp
 from .auth import is_api_authd
-from .support import error, handleCreateRelease
+from .support import error, api_create_vcs_release, api_create_zip_release
 from app import csrf
 from app.models import *
 from app.utils import is_package_page
@@ -210,15 +210,23 @@ def create_release(token, package):
 	if not package.checkPerm(token.owner, Permission.APPROVE_RELEASE):
 		error(403, "You do not have the permission to approve releases")
 
-	json = request.json
+	data = request.json or request.form
-	if json is None:
+	if "title" not in data:
-		error(400, "JSON post data is required")
+		error(400, "Title is required in the POST data")
 
-	for option in ["method", "title", "ref"]:
+	if request.json:
-		if json.get(option) is None:
+		for option in ["method", "ref"]:
-			error(400, option + " is required in the POST data")
+			if option not in data:
+				error(400, option + " is required in the POST data")
 
-	if json["method"].lower() != "git":
+		if data["method"].lower() != "git":
-		error(400, "Release-creation methods other than git are not supported")
+			error(400, "Release-creation methods other than git are not supported")
 
-	return handleCreateRelease(token, package, json["title"], json["ref"])
+		return api_create_vcs_release(token, package, data["title"], data["ref"])
+
+	elif request.files:
+		file = request.files.get("file")
+		if file is None:
+			error(400, "Missing 'file' in multipart body")
+
+		return api_create_zip_release(token, package, data["title"], file)

app/blueprints/api/support.py

@@ -1,44 +1,55 @@
-import datetime
+# ContentDB
+# Copyright (C) 2018-21 rubenwardy
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
 
-from celery import uuid
 from flask import jsonify, abort, make_response, url_for
-
+from app.logic.releases import LogicError, do_create_vcs_release, do_create_zip_release
-from app.models import PackageRelease, db, Permission
+from app.models import APIToken, Package, MinetestRelease
-from app.tasks.importtasks import makeVCSRelease
-from app.utils import AuditSeverity, addAuditLog
 
 
-def error(status, message):
+def error(code: int, msg: str):
-	abort(make_response(jsonify({ "success": False, "error": message }), status))
+	abort(make_response(jsonify({ "success": False, "error": msg }), code))
+
+# Catches LogicErrors and aborts with JSON error
+def run_safe(f, *args, **kwargs):
+	try:
+		return f(*args, **kwargs)
+	except LogicError as e:
+		error(e.code, e.message)
 
 
-def handleCreateRelease(token, package, title, ref, reason="API"):
+def api_create_vcs_release(token: APIToken, package: Package, title: str, ref: str,
+		min_v: MinetestRelease = None, max_v: MinetestRelease = None, reason="API"):
 	if not token.canOperateOnPackage(package):
-		return error(403, "API token does not have access to the package")
+		error(403, "API token does not have access to the package")
 
-	if not package.checkPerm(token.owner, Permission.MAKE_RELEASE):
+	rel = run_safe(do_create_vcs_release, token.owner, package, title, ref, None, None, reason)
-		return error(403, "Permission denied. Missing MAKE_RELEASE permission")
+
-
+	return jsonify({
-	five_minutes_ago = datetime.datetime.now() - datetime.timedelta(minutes=5)
+		"success": True,
-	count = package.releases.filter(PackageRelease.releaseDate > five_minutes_ago).count()
+		"task": url_for("tasks.check", id=rel.task_id),
-	if count >= 2:
+		"release": rel.getAsDictionary()
-		return error(429, "Too many requests, please wait before trying again")
+	})
-
+
-	rel = PackageRelease()
+
-	rel.package = package
+def api_create_zip_release(token: APIToken, package: Package, title: str, file, reason="API"):
-	rel.title   = title
+	if not token.canOperateOnPackage(package):
-	rel.url     = ""
+		error(403, "API token does not have access to the package")
-	rel.task_id = uuid()
+
-	rel.min_rel = None
+	rel = run_safe(do_create_zip_release, token.owner, package, title, file, None, None, reason)
-	rel.max_rel = None
-	db.session.add(rel)
-
-	msg = "Created release {} ({})".format(rel.title, reason)
-	addAuditLog(AuditSeverity.NORMAL, token.owner, msg, package.getDetailsURL(), package)
-
-	db.session.commit()
-
-	makeVCSRelease.apply_async((rel.id, ref), task_id=rel.task_id)
 
 	return jsonify({
 		"success": True,

app/blueprints/github/__init__.py

@@ -24,7 +24,7 @@ from sqlalchemy import func, or_, and_
 from app import github, csrf
 from app.models import db, User, APIToken, Package, Permission, AuditSeverity
 from app.utils import abs_url_for, addAuditLog, login_user_set_active
-from app.blueprints.api.support import error, handleCreateRelease
+from app.blueprints.api.support import error, api_create_vcs_release
 import hmac, requests
 
 @bp.route("/github/start/")
@@ -146,4 +146,4 @@ def webhook():
 	# Perform release
 	#
 
-	return handleCreateRelease(actual_token, package, title, ref, reason="Webhook")
+	return api_create_vcs_release(actual_token, package, title, ref, reason="Webhook")

app/blueprints/gitlab/__init__.py

@@ -20,7 +20,7 @@ bp = Blueprint("gitlab", __name__)
 
 from app import csrf
 from app.models import Package, APIToken, Permission
-from app.blueprints.api.support import error, handleCreateRelease
+from app.blueprints.api.support import error, api_create_vcs_release
 
 
 def webhook_impl():
@@ -63,7 +63,7 @@ def webhook_impl():
 	# Perform release
 	#
 
-	return handleCreateRelease(token, package, title, ref, reason="Webhook")
+	return api_create_vcs_release(token, package, title, ref, reason="Webhook")
 
 
 @bp.route("/gitlab/webhook/", methods=["POST"])

app/blueprints/packages/releases.py

@@ -15,7 +15,6 @@
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
 
-from celery import uuid
 from flask import *
 from flask_login import login_required
 from flask_wtf import FlaskForm
@@ -23,8 +22,9 @@ from wtforms import *
 from wtforms.ext.sqlalchemy.fields import QuerySelectField
 from wtforms.validators import *
 
+from app.logic.releases import do_create_vcs_release, LogicError, do_create_zip_release
 from app.rediscache import has_key, set_key, make_download_key
-from app.tasks.importtasks import makeVCSRelease, checkZipRelease, check_update_config
+from app.tasks.importtasks import check_update_config
 from app.utils import *
 from . import bp
 
@@ -80,48 +80,16 @@ def create_release(package):
 		form.title.data = request.args.get("title")
 
 	if form.validate_on_submit():
-		if form["uploadOpt"].data == "vcs":
+		try:
-			rel = PackageRelease()
+			if form["uploadOpt"].data == "vcs":
-			rel.package = package
+				rel = do_create_vcs_release(current_user, package, form.title.data,
-			rel.title   = form["title"].data
+						form.vcsLabel.data, form.min_rel.data.getActual(), form.max_rel.data.getActual())
-			rel.url     = ""
+			else:
-			rel.task_id = uuid()
+				rel = do_create_zip_release(current_user, package, form.title.data,
-			rel.min_rel = form["min_rel"].data.getActual()
+						form.fileUpload.data, form.min_rel.data.getActual(), form.max_rel.data.getActual())
-			rel.max_rel = form["max_rel"].data.getActual()
-			db.session.add(rel)
-			db.session.commit()
-
-			makeVCSRelease.apply_async((rel.id, nonEmptyOrNone(form.vcsLabel.data)), task_id=rel.task_id)
-
-			msg = "Created release {}".format(rel.title)
-			addNotification(package.maintainers, current_user, NotificationType.PACKAGE_EDIT, msg, rel.getEditURL(), package)
-			addAuditLog(AuditSeverity.NORMAL, current_user, msg, package.getDetailsURL(), package)
-			db.session.commit()
-
 			return redirect(url_for("tasks.check", id=rel.task_id, r=rel.getEditURL()))
-		else:
+		except LogicError as e:
-			uploadedUrl, uploadedPath = doFileUpload(form.fileUpload.data, "zip", "a zip file")
+			flash(e.message, "danger")
-			if uploadedUrl is not None:
-				rel = PackageRelease()
-				rel.package = package
-				rel.title = form["title"].data
-				rel.url = uploadedUrl
-				rel.task_id = uuid()
-				rel.min_rel = form["min_rel"].data.getActual()
-				rel.max_rel = form["max_rel"].data.getActual()
-				db.session.add(rel)
-				db.session.commit()
-
-				checkZipRelease.apply_async((rel.id, uploadedPath), task_id=rel.task_id)
-
-				msg = "Created release {}".format(rel.title)
-				addNotification(package.maintainers, current_user, NotificationType.PACKAGE_EDIT,
-						msg, rel.getEditURL(), package)
-				addAuditLog(AuditSeverity.NORMAL, current_user, msg, package.getDetailsURL(),
-						package)
-				db.session.commit()
-
-				return redirect(url_for("tasks.check", id=rel.task_id, r=rel.getEditURL()))
 
 	return render_template("packages/release_new.html", package=package, form=form)
 

app/blueprints/packages/screenshots.py

@@ -24,6 +24,8 @@ from wtforms.validators import *
 
 from app.utils import *
 from . import bp
+from app.logic.LogicError import LogicError
+from app.logic.screenshots import do_create_screenshot
 
 
 class CreateScreenshotForm(FlaskForm):
@@ -88,27 +90,11 @@ def create_screenshot(package):
 	# Initial form class from post data and default data
 	form = CreateScreenshotForm()
 	if form.validate_on_submit():
-		uploadedUrl, uploadedPath = doFileUpload(form.fileUpload.data, "image",
+		try:
-				"a PNG or JPG image file")
+			do_create_screenshot(current_user, package, form.title.data, form.fileUpload.data)
-		if uploadedUrl is not None:
-			counter = 1
-			for screenshot in package.screenshots:
-				screenshot.order = counter
-				counter += 1
-
-			ss = PackageScreenshot()
-			ss.package  = package
-			ss.title    = form["title"].data or "Untitled"
-			ss.url      = uploadedUrl
-			ss.approved = package.checkPerm(current_user, Permission.APPROVE_SCREENSHOT)
-			ss.order    = counter
-			db.session.add(ss)
-
-			msg = "Screenshot added {}" \
-					.format(ss.title)
-			addNotification(package.maintainers, current_user, NotificationType.PACKAGE_EDIT, msg, package.getDetailsURL(), package)
-			db.session.commit()
 			return redirect(package.getEditScreenshotsURL())
+		except LogicError as e:
+			flash(e.message, "danger")
 
 	return render_template("packages/screenshot_new.html", package=package, form=form)
 

app/flatpages/help/api.md

@@ -46,10 +46,13 @@ Tokens can be attained by visiting [Profile > "API Tokens"](/user/tokens/).
 * GET `/api/packages/<username>/<name>/releases/`
 * POST `/api/packages/<username>/<name>/releases/new/`
 	* Requires authentication.
+	* Body is multipart form if zip upload, JSON otherwise.
 	* `title`: human-readable name of the release.
-	* `method`: Release-creation method, only `git` is supported.
+	* For Git release creation:
-	* If `git` release-creation method:
+		* `method`: must be `git`.
-		* `ref` - git reference, eg: `master`.
+		* `ref`: (Optional) git reference, eg: `master`.
+	* For zip upload release creation: 
+		* `file`: multipart file to upload, like `<input type=file>`.
 	* You can set min and max Minetest Versions [using the content's .conf file](/help/package_config/).
 
 

app/logic/LogicError.py

@@ -0,0 +1,24 @@
+# ContentDB
+# Copyright (C) 2021 rubenwardy
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+
+class LogicError(Exception):
+	def __init__(self, code, message):
+		self.code = code
+		self.message = message
+
+	def __str__(self):
+		return repr("LogicError {}: {}".format(self.code, self.message))

app/logic/releases.py

@@ -0,0 +1,90 @@
+# ContentDB
+# Copyright (C) 2018-21 rubenwardy
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+
+import datetime
+
+from celery import uuid
+
+from app.logic.LogicError import LogicError
+from app.logic.uploads import upload_file
+from app.models import PackageRelease, db, Permission, User, Package, MinetestRelease
+from app.tasks.importtasks import makeVCSRelease, checkZipRelease
+from app.utils import AuditSeverity, addAuditLog, nonEmptyOrNone
+
+
+def check_can_create_release(user: User, package: Package):
+	if not package.checkPerm(user, Permission.MAKE_RELEASE):
+		raise LogicError(403, "Permission denied. Missing MAKE_RELEASE permission")
+
+	five_minutes_ago = datetime.datetime.now() - datetime.timedelta(minutes=5)
+	count = package.releases.filter(PackageRelease.releaseDate > five_minutes_ago).count()
+	if count >= 2:
+		raise LogicError(429, "Too many requests, please wait before trying again")
+
+
+def do_create_vcs_release(user: User, package: Package, title: str, ref: str,
+		min_v: MinetestRelease = None, max_v: MinetestRelease = None, reason: str = None):
+	check_can_create_release(user, package)
+
+	rel = PackageRelease()
+	rel.package = package
+	rel.title   = title
+	rel.url     = ""
+	rel.task_id = uuid()
+	rel.min_rel = min_v
+	rel.max_rel = max_v
+	db.session.add(rel)
+
+	if reason is None:
+		msg = "Created release {}".format(rel.title)
+	else:
+		msg = "Created release {} ({})".format(rel.title, reason)
+	addAuditLog(AuditSeverity.NORMAL, user, msg, package.getDetailsURL(), package)
+
+	db.session.commit()
+
+	makeVCSRelease.apply_async((rel.id, nonEmptyOrNone(ref)), task_id=rel.task_id)
+
+	return rel
+
+
+def do_create_zip_release(user: User, package: Package, title: str, file,
+		min_v: MinetestRelease = None, max_v: MinetestRelease = None, reason: str = None):
+	check_can_create_release(user, package)
+
+	uploaded_url, uploaded_path = upload_file(file, "zip", "a zip file")
+
+	rel = PackageRelease()
+	rel.package = package
+	rel.title   = title
+	rel.url     = uploaded_url
+	rel.task_id = uuid()
+	rel.min_rel = min_v
+	rel.max_rel = max_v
+	db.session.add(rel)
+
+	if reason is None:
+		msg = "Created release {}".format(rel.title)
+	else:
+		msg = "Created release {} ({})".format(rel.title, reason)
+	addAuditLog(AuditSeverity.NORMAL, user, msg, package.getDetailsURL(), package)
+
+	db.session.commit()
+
+	checkZipRelease.apply_async((rel.id, uploaded_path), task_id=rel.task_id)
+
+	return rel

app/logic/screenshots.py

@@ -0,0 +1,29 @@
+from werkzeug.exceptions import abort
+
+from app.logic.uploads import upload_file
+from app.models import User, Package, PackageScreenshot, Permission, NotificationType, db
+from app.utils import addNotification
+
+
+def do_create_screenshot(user: User, package: Package, title: str, file):
+	uploaded_url, uploaded_path = upload_file(file, "image", "a PNG or JPG image file")
+
+	counter = 1
+	for screenshot in package.screenshots:
+		screenshot.order = counter
+		counter += 1
+
+	ss = PackageScreenshot()
+	ss.package  = package
+	ss.title    = title or "Untitled"
+	ss.url      = uploaded_url
+	ss.approved = package.checkPerm(user, Permission.APPROVE_SCREENSHOT)
+	ss.order    = counter
+	db.session.add(ss)
+
+	msg = "Screenshot added {}" \
+			.format(ss.title)
+	addNotification(package.maintainers, user, NotificationType.PACKAGE_EDIT, msg, package.getDetailsURL(), package)
+	db.session.commit()
+
+	return ss

app/logic/uploads.py

@@ -17,37 +17,25 @@
 
 import imghdr
 import os
-import random
-import string
-
-from flask import request, flash
 
+from app.logic.LogicError import LogicError
 from app.models import *
+from app.utils import randomString
 
 
-def getExtension(filename):
+def get_extension(filename):
 	return filename.rsplit(".", 1)[1].lower() if "." in filename else None
 
 ALLOWED_IMAGES = {"jpeg", "png"}
 def isAllowedImage(data):
 	return imghdr.what(None, data) in ALLOWED_IMAGES
 
-def shouldReturnJson():
+def upload_file(file, fileType, fileTypeDesc):
-	return "application/json" in request.accept_mimetypes and \
-			not "text/html" in request.accept_mimetypes
-
-def randomString(n):
-	return ''.join(random.choice(string.ascii_lowercase + \
-			string.ascii_uppercase + string.digits) for _ in range(n))
-
-def doFileUpload(file, fileType, fileTypeDesc):
 	if not file or file is None or file.filename == "":
-		flash("No selected file", "danger")
+		raise LogicError(400, "Expected file")
-		return None, None
 
 	assert os.path.isdir(app.config["UPLOAD_DIR"]), "UPLOAD_DIR must exist"
 
-	allowedExtensions = []
 	isImage = False
 	if fileType == "image":
 		allowedExtensions = ["jpg", "jpeg", "png"]
@@ -57,18 +45,17 @@ def doFileUpload(file, fileType, fileTypeDesc):
 	else:
 		raise Exception("Invalid fileType")
 
-	ext = getExtension(file.filename)
+	ext = get_extension(file.filename)
 	if ext is None or not ext in allowedExtensions:
-		flash("Please upload " + fileTypeDesc, "danger")
+		raise LogicError(400, "Please upload " + fileTypeDesc)
-		return None, None
 
 	if isImage and not isAllowedImage(file.stream.read()):
-		flash("Uploaded image isn't actually an image", "danger")
+		raise LogicError(400, "Uploaded image isn't actually an image")
-		return None, None
 
 	file.stream.seek(0)
 
 	filename = randomString(10) + "." + ext
 	filepath = os.path.join(app.config["UPLOAD_DIR"], filename)
 	file.save(filepath)
+
 	return "/uploads/" + filename, filepath

app/utils/__init__.py

@@ -13,24 +13,37 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-
+import random
+import string
 
 from .flask import *
-from .uploads import *
 from .models import *
 from .user import *
 
 
 YESES = ["yes", "true", "1", "on"]
 
+
 def isYes(val):
 	return val and val.lower() in YESES
 
+
 def isNo(val):
 	return val and not isYes(val)
 
+
 def nonEmptyOrNone(str):
 	if str is None or str == "":
 		return None
 
 	return str
+
+
+def shouldReturnJson():
+	return "application/json" in request.accept_mimetypes and \
+			not "text/html" in request.accept_mimetypes
+
+
+def randomString(n):
+	return ''.join(random.choice(string.ascii_lowercase + \
+			string.ascii_uppercase + string.digits) for _ in range(n))