Add package update configuration for polling

app/blueprints/packages/releases.py

@@ -24,7 +24,7 @@ from wtforms.ext.sqlalchemy.fields import QuerySelectField
 from wtforms.validators import *
 
 from app.rediscache import has_key, set_key, make_download_key
-from app.tasks.importtasks import makeVCSRelease, checkZipRelease
+from app.tasks.importtasks import makeVCSRelease, checkZipRelease, updateMetaFromRelease, check_for_updates
 from app.utils import *
 from . import bp
 
@@ -247,3 +247,34 @@ def delete_release(package, id):
 	db.session.commit()
 
 	return redirect(package.getDetailsURL())
+
+
+class PackageUpdateConfigFrom(FlaskForm):
+	trigger = SelectField("Trigger", [InputRequired()], choices=PackageUpdateTrigger.choices(), coerce=PackageUpdateTrigger.coerce,
+			default=PackageUpdateTrigger.COMMIT)
+	make_release = BooleanField("Create Release")
+	submit	   = SubmitField("Save")
+
+
+@bp.route("/packages/<author>/<name>/update-config/", methods=["GET", "POST"])
+@login_required
+@is_package_page
+def update_config(package):
+	package.update_config = package.update_config or PackageUpdateConfig()
+
+	if not package.checkPerm(current_user, Permission.MAKE_RELEASE):
+		return redirect(package.getDetailsURL())
+
+	form = PackageUpdateConfigFrom(obj=package.update_config)
+	if form.validate_on_submit():
+		flash("Changed update configuration", "success")
+		form.populate_obj(package.update_config)
+		db.session.add(package.update_config)
+
+		check_for_updates.delay()
+
+		db.session.commit()
+
+		return redirect(package.getDetailsURL())
+
+	return render_template("packages/update_config.html", package=package, form=form)

app/models/packages.py

@@ -317,7 +317,7 @@ class Package(db.Model):
 	maintainers = db.relationship("User", secondary=maintainers)
 
 	threads = db.relationship("Thread", back_populates="package", order_by=db.desc("thread_created_at"),
-			foreign_keys="Thread.package_id", cascade="all, delete, delete-orphan")
+			foreign_keys="Thread.package_id", cascade="all, delete, delete-orphan", lazy="dynamic")
 
 	reviews = db.relationship("PackageReview", back_populates="package", order_by=db.desc("package_review_created_at"),
 			cascade="all, delete, delete-orphan")
@@ -331,6 +331,9 @@ class Package(db.Model):
 	tokens = db.relationship("APIToken", foreign_keys="APIToken.package_id", back_populates="package",
 			cascade="all, delete, delete-orphan")
 
+	update_config = db.relationship("PackageUpdateConfig", uselist=False, back_populates="package",
+			cascade="all, delete, delete-orphan")
+
 	def __init__(self, package=None):
 		if package is None:
 			return
@@ -507,6 +510,10 @@ class Package(db.Model):
 		return url_for("packages.bulk_change_release",
 			author=self.author.username, name=self.name)
 
+	def getUpdateConfigURL(self):
+		return url_for("packages.update_config",
+			author=self.author.username, name=self.name)
+
 	def getDownloadURL(self):
 		return url_for("packages.download",
 				author=self.author.username, name=self.name)
@@ -790,7 +797,7 @@ class PackageRelease(db.Model):
 
 	title        = db.Column(db.String(100), nullable=False)
 	releaseDate  = db.Column(db.DateTime,    nullable=False)
-	url          = db.Column(db.String(200), nullable=False)
+	url          = db.Column(db.String(200), nullable=False, default="")
 	approved     = db.Column(db.Boolean, nullable=False, default=False)
 	task_id      = db.Column(db.String(37), nullable=True)
 	commit_hash  = db.Column(db.String(41), nullable=True, default=None)
@@ -903,3 +910,39 @@ class PackageScreenshot(db.Model):
 
 	def getThumbnailURL(self, level=2):
 		return self.url.replace("/uploads/", "/thumbnails/{:d}/".format(level))
+
+
+class PackageUpdateTrigger(enum.Enum):
+	COMMIT = "New Commit"
+	TAG = "New Tag"
+
+	def toName(self):
+		return self.name.lower()
+
+	def __str__(self):
+		return self.name
+
+	@classmethod
+	def get(cls, name):
+		try:
+			return PackageUpdateTrigger[name.upper()]
+		except KeyError:
+			return None
+
+	@classmethod
+	def choices(cls):
+		return [(choice, choice.value) for choice in cls]
+
+	@classmethod
+	def coerce(cls, item):
+		return item if type(item) == PackageUpdateTrigger else PackageUpdateTrigger[item]
+
+
+class PackageUpdateConfig(db.Model):
+	package_id  = db.Column(db.Integer, db.ForeignKey("package.id"), primary_key=True)
+	package     = db.relationship("Package", back_populates="update_config", foreign_keys=[package_id])
+
+	last_commit = db.Column(db.String(41), nullable=True, default=None)
+
+	trigger     = db.Column(db.Enum(PackageUpdateTrigger), nullable=False, default=PackageUpdateTrigger.COMMIT)
+	make_release = db.Column(db.Boolean, nullable=False, default=False)

app/models/threads.py

@@ -56,7 +56,7 @@ class Thread(db.Model):
 	watchers   = db.relationship("User", secondary=watchers, backref="watching")
 
 	def getViewURL(self):
-		return url_for("threads.view", id=self.id)
+		return url_for("threads.view", id=self.id, _external=False)
 
 	def getSubscribeURL(self):
 		return url_for("threads.subscribe", id=self.id)

app/models/users.py

@@ -32,8 +32,9 @@ class UserRank(enum.Enum):
 	MEMBER         = 3
 	TRUSTED_MEMBER = 4
 	EDITOR         = 5
-	MODERATOR      = 6
+	BOT            = 6
-	ADMIN          = 7
+	MODERATOR      = 7
+	ADMIN          = 8
 
 	def atLeast(self, min):
 		return self.value >= min.value
@@ -192,6 +193,8 @@ class User(db.Model, UserMixin):
 	def getProfilePicURL(self):
 		if self.profile_pic:
 			return self.profile_pic
+		elif self.rank == UserRank.BOT:
+			return "/static/bot_avatar.png"
 		else:
 			return gravatar(self.email or "")
 

app/scss/comments.scss

@@ -26,4 +26,10 @@
 			border-width: 14px;
 		}
 	}
+
+	.user-photo {
+		width: 60px;
+		height: 60px;
+		object-fit: cover;
+	}
 }

app/scss/components.scss

@@ -81,6 +81,10 @@
 	color: #2c2 !important;
 }
 
+.BOT a, .BOT {
+	color: #FFDF00 !important;
+}
+
 .wiptopic a:not(.btn) {
 	color: #7ac;
 }

app/tasks/__init__.py

@@ -73,6 +73,11 @@ CELERYBEAT_SCHEDULE = {
 		'task': 'app.tasks.pkgtasks.updatePackageScores',
 		'schedule': crontab(minute=10, hour=1),
 	},
+
+	'package_score_update': {
+		'task': 'app.tasks.importtasks.check_for_updates',
+		'schedule': crontab(minute=10, hour=1),
+	},
 	'send_pending_notifications': {
 		'task': 'app.tasks.emails.send_pending_notifications',
 		'schedule': crontab(minute='*/5'),

app/tasks/importtasks.py

@@ -22,9 +22,12 @@ from urllib.error import HTTPError
 import urllib.request
 from urllib.parse import urlsplit
 from zipfile import ZipFile
+
+from kombu import uuid
+
 from app.models import *
 from app.tasks import celery, TaskError
-from app.utils import randomString, getExtension
+from app.utils import randomString, getExtension, post_system_thread
 from .minetestcheck import build_tree, MinetestCheckError, ContentType
 
 
@@ -85,6 +88,40 @@ def clone_repo(urlstr, ref=None, recursive=False):
 			.strip())
 
 
+def get_commit_hash(urlstr, ref=None):
+	gitDir = os.path.join(tempfile.gettempdir(), randomString(10))
+
+	err = None
+	try:
+		gitUrl = generateGitURL(urlstr)
+		print("Cloning from " + gitUrl)
+
+		assert ref != ""
+
+		repo = git.Repo.init(gitDir)
+		origin: git.Remote = repo.create_remote("origin", url=gitUrl)
+		assert origin.exists()
+		origin.fetch()
+
+		if ref:
+			ref: git.Reference = origin.refs[ref]
+		else:
+			ref: git.Reference = origin.refs[0]
+
+		return ref.commit.hexsha
+
+	except GitCommandError as e:
+		# This is needed to stop the backtrace being weird
+		err = e.stderr
+
+	except gitdb.exc.BadName as e:
+		err = "Unable to find the reference " + (ref or "?") + "\n" + e.stderr
+
+	raise TaskError(err.replace("stderr: ", "") \
+		.replace("Cloning into '" + gitDir + "'...", "") \
+		.strip())
+
+
 @celery.task()
 def getMeta(urlstr, author):
 	with clone_repo(urlstr, recursive=True) as repo:
@@ -274,3 +311,50 @@ def importForeignDownloads(self, id):
 		release.task_id = self.request.id
 		release.approved = False
 		db.session.commit()
+
+
+@celery.task
+def check_update_config(package_id):
+	package: Package = Package.query.get(package_id)
+	if package is None:
+		raise TaskError("No such package!")
+	elif package.update_config is None:
+		raise TaskError("No update config attached to package")
+
+	config = package.update_config
+	ref = None
+	hash = get_commit_hash(package.repo, ref)
+
+	if config.last_commit != hash:
+		if config.make_release:
+			rel = PackageRelease()
+			rel.package = package
+			rel.title = hash[0:5]
+			rel.url = ""
+			rel.task_id = uuid()
+			db.session.add(rel)
+			db.session.commit()
+
+			makeVCSRelease.apply_async((rel.id, ref), task_id=rel.task_id)
+
+		else:
+			post_system_thread(package, "New commit detected, package outdated?",
+					"Commit {} was detected on the Git repository.\n\n[Change update configuration]({})" \
+						.format(hash[0:5], package.getUpdateConfigURL()))
+
+	config.last_commit = hash
+	db.session.commit()
+
+
+@celery.task
+def check_for_updates():
+	for update_config in PackageUpdateConfig.query.all():
+		update_config: PackageUpdateConfig
+
+		if update_config.package.repo is None:
+			db.session.delete(update_config)
+			continue
+
+		check_update_config.delay(update_config.package_id)
+
+	db.session.commit()

app/templates/macros/threads.html

@@ -11,10 +11,22 @@
 		<div class="col pr-0">
 			<div class="card">
 				<div class="card-header">
-					<a class="author {{ r.author.rank.name }}"
+					<a class="author {{ r.author.rank.name }} mr-3"
 							href="{{ url_for('users.profile', username=r.author.username) }}">
 						{{ r.author.display_name }}
 					</a>
+
+					{% if r.author in thread.package.maintainers %}
+						<span class="badge badge-dark">
+							{{ _("Maintainer") }}
+						</span>
+					{% endif %}
+					{% if r.author.rank == r.author.rank.BOT %}
+						<span class="badge badge-dark">
+							{{ r.author.rank.getTitle() }}
+						</span>
+					{% endif %}
+
 					<a name="reply-{{ r.id }}" class="text-muted float-right"
 							href="{{ url_for('threads.view', id=thread.id) }}#reply-{{ r.id }}">
 						{{ r.created_at | datetime }}

app/templates/packages/update_config.html

@@ -0,0 +1,29 @@
+{% extends "base.html" %}
+
+{% block title %}
+	Configure Update Detection | {{ package.title }}
+{% endblock %}
+
+{% block content %}
+	<h1>{{ _("Configure Update Detection") }}</h1>
+
+	<p>
+		{{ _("ContentDB will poll your Git repository at 2am UTC every day.") }}
+		{{ _("You should consider using webhooks or the API for faster rollouts.") }}
+	</p>
+
+	{% from "macros/forms.html" import render_field, render_submit_field, render_checkbox_field %}
+	<form method="POST" action="">
+		{{ form.hidden_tag() }}
+
+		<h3>Triggers</h3>
+		{{ render_field(form.trigger) }}
+
+		<h3 class="mt-5">Actions</h3>
+		{{ render_checkbox_field(form.make_release) }}
+
+		<p class="mt-5">
+			{{ render_submit_field(form.submit) }}
+		</p>
+	</form>
+{% endblock %}

app/templates/users/list.html

@@ -38,6 +38,8 @@
 						<i class="fas fa-user-shield mr-2"></i>
 					{% elif user.rank == user.rank.EDITOR %}
 						<i class="fas fa-user-edit mr-2"></i>
+					{% elif user.rank == user.rank.BOT %}
+						<i class="fas fa-robot mr-2"></i>
 					{% else %}
 						<i class="fas fa-user mr-2"></i>
 					{% endif %}

app/templates/users/profile.html

@@ -23,7 +23,7 @@
 			</a>
 		{% endif %}
 
-		<h1 class="ml-3 my-0">
+		<h1 class="ml-3 my-0 {{ user.rank.name }}">
 			{{ user.display_name }}
 		</h1>
 

app/utils.py

@@ -253,3 +253,30 @@ def nonEmptyOrNone(str):
 		return None
 
 	return str
+
+
+def post_system_thread(package: Package, title: str, message: str):
+	system_user = User.query.filter_by(username="ContentDB").first()
+	assert system_user
+
+	thread = package.threads.filter_by(author=system_user).first()
+	if not thread:
+		thread = Thread()
+		thread.package = package
+		thread.title = "System Notifications"
+		thread.author = system_user
+		thread.private = True
+		thread.watchers.append(package.author)
+		db.session.add(thread)
+		db.session.flush()
+
+	reply = ThreadReply()
+	reply.thread  = thread
+	reply.author  = system_user
+	reply.comment = "# {}\n\n{}".format(title, message)
+	db.session.add(reply)
+
+	addNotification(thread.watchers, system_user, NotificationType.THREAD_REPLY,
+			title, thread.getViewURL(), thread.package)
+
+	thread.replies.append(reply)

migrations/versions/105d4c740ad6_.py

@@ -0,0 +1,36 @@
+"""empty message
+
+Revision ID: 105d4c740ad6
+Revises: 886c92dc6eaa
+Create Date: 2020-12-15 17:28:56.559801
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+from sqlalchemy import orm
+from app.models import User, UserRank
+
+revision = '105d4c740ad6'
+down_revision = '886c92dc6eaa'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.execute("COMMIT")
+    op.execute("ALTER TYPE userrank ADD VALUE 'BOT' AFTER 'EDITOR'")
+
+    conn = op.get_bind()
+    system_user = User("ContentDB", active=False)
+    system_user.rank = UserRank.BOT
+
+    session = orm.Session(bind=conn)
+    session.add(system_user)
+    session.commit()
+
+
+def downgrade():
+    pass

migrations/versions/886c92dc6eaa_.py

@@ -0,0 +1,35 @@
+"""empty message
+
+Revision ID: 886c92dc6eaa
+Revises: 8d22def23c8b
+Create Date: 2020-12-15 16:38:54.114559
+
+"""
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = '886c92dc6eaa'
+down_revision = '8d22def23c8b'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table('package_update_config',
+    sa.Column('package_id', sa.Integer(), nullable=False),
+    sa.Column('last_commit', sa.String(length=41), nullable=True),
+    sa.Column('trigger', sa.Enum('COMMIT', 'TAG', name='packageupdatetrigger'), nullable=False),
+    sa.Column('make_release', sa.Boolean(), nullable=False),
+    sa.ForeignKeyConstraint(['package_id'], ['package.id'], ),
+    sa.PrimaryKeyConstraint('package_id')
+    )
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_table('package_update_config')
+    # ### end Alembic commands ###

utils/create_migration.sh

@@ -2,7 +2,7 @@
 
 # Create a database migration, and copy it back to the host.
 
-docker exec contentdb_app_1 sh -c "FLASK_CONFIG=../config.cfg FLASK_APP=app/__init__.py flask db migrate"
+docker exec contentdb_app_1 sh -c "FLASK_CONFIG=../config.cfg FLASK_APP=app/__init__.py flask db revision"
 docker exec -u root contentdb_app_1 sh -c "cp /home/cdb/migrations/versions/* /source/migrations/versions/"
 
 USER=$(whoami)