Add ability to search for text in all packages

app/blueprints/tasks/__init__.py

@@ -25,6 +25,7 @@ from app.utils import *
 
 bp = Blueprint("tasks", __name__)
 
+
 @csrf.exempt
 @bp.route("/tasks/getmeta/new/", methods=["POST"])
 @login_required
@@ -36,6 +37,7 @@ def start_getmeta():
 		"poll_url": url_for("tasks.check", id=aresult.id),
 	})
 
+
 @bp.route("/tasks/<id>/")
 def check(id):
 	result = celery.AsyncResult(id)
@@ -43,7 +45,6 @@ def check(id):
 	traceback = result.traceback
 	result = result.result
 
-	None
 	if isinstance(result, Exception):
 		info = {
 				'id': id,

app/blueprints/zipgrep/__init__.py

@@ -0,0 +1,66 @@
+# ContentDB
+# Copyright (C) 2022 rubenwardy
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+
+from celery import uuid
+from flask import Blueprint, render_template, redirect, request
+from flask_wtf import FlaskForm
+from wtforms import StringField, BooleanField, SubmitField
+from wtforms.validators import InputRequired, Length
+
+from app.tasks import celery
+from app.utils import rank_required
+
+bp = Blueprint("zipgrep", __name__)
+
+from app.models import *
+from app.tasks.zipgrep import search_in_releases
+
+
+class SearchForm(FlaskForm):
+	query = StringField(lazy_gettext("Text to find (regex)"), [InputRequired(), Length(6, 100)])
+	file_filter = StringField(lazy_gettext("File filter"), [InputRequired(), Length(1, 100)], default="*.lua")
+	remember_me = BooleanField(lazy_gettext("Remember me"), default=True)
+	submit = SubmitField(lazy_gettext("Search"))
+
+
+@bp.route("/zipgrep/", methods=["GET", "POST"])
+@rank_required(UserRank.ADMIN)
+def zipgrep_search():
+	form = SearchForm(request.form)
+	if form.validate_on_submit():
+		task_id = uuid()
+		search_in_releases.apply_async((form.query.data, form.file_filter.data), task_id=task_id)
+		result_url = url_for("zipgrep.view_results", id=task_id)
+		return redirect(url_for("tasks.check", id=task_id, r=result_url))
+
+	return render_template("zipgrep/search.html", form=form)
+
+
+@bp.route("/zipgrep/<id>/")
+def view_results(id):
+	result = celery.AsyncResult(id)
+	if result.status != "SUCCESS" or isinstance(result.result, Exception):
+		result_url = url_for("zipgrep.view_results", id=id)
+		return redirect(url_for("tasks.check", id=id, r=result_url))
+
+	matches = result.result["matches"]
+	for match in matches:
+		match["package"] = Package.query.filter(
+				Package.name == match["package"]["name"],
+				Package.author.has(username=match["package"]["author"])).one()
+
+	return render_template("zipgrep/view_results.html", query=result.result["query"], matches=matches)

app/tasks/zipgrep.py

@@ -0,0 +1,62 @@
+# ContentDB
+# Copyright (C) 2022 rubenwardy
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+
+import subprocess
+from subprocess import Popen, PIPE
+from typing import Optional
+
+from app.models import Package, PackageState, PackageRelease
+from app.tasks import celery
+
+
+@celery.task()
+def search_in_releases(query: str, file_filter: str):
+	packages = list(Package.query.filter(Package.state == PackageState.APPROVED).all())
+	running = []
+	results = []
+
+	while len(packages) > 0 or len(running) > 0:
+		# Check running
+		for i in range(len(running) - 1, -1, -1):
+			package: Package = running[i][0]
+			handle: subprocess.Popen[str] = running[i][1]
+			exit_code = handle.poll()
+			if exit_code is None:
+				continue
+			elif exit_code == 0:
+				results.append({
+					"package": package.getAsDictionaryKey(),
+					"lines": handle.stdout.read(),
+				})
+
+			del running[i]
+
+		# Create new
+		while len(running) < 1 and len(packages) > 0:
+			package = packages.pop()
+			release: Optional[PackageRelease] = package.getDownloadRelease()
+			if release:
+				handle = Popen(["zipgrep", query, release.file_path, file_filter], stdout=PIPE, encoding="UTF-8")
+				running.append([package, handle])
+
+		if len(running) > 0:
+			running[0][1].wait()
+
+	return {
+		"query": query,
+		"matches": results,
+	}

app/templates/zipgrep/search.html

@@ -0,0 +1,26 @@
+{% extends "base.html" %}
+
+{% block title %}
+{{ _("Search in Package Releases") }}
+{% endblock %}
+
+{% block query_hint %}
+<a href="https://www.digitalocean.com/community/tutorials/using-grep-regular-expressions-to-search-for-text-patterns-in-linux#extended-regular-expressions">
+	POSIX Extended Regular Expressions
+</a>
+{% endblock %}
+
+{% block content %}
+	<h1>{{ self.title() }}</h1>
+	{% from "macros/forms.html" import render_field, render_submit_field %}
+	<form action="" method="POST" class="form" role="form">
+		{{ form.hidden_tag() }}
+		{{ render_field(form.query, hint=self.query_hint()) }}
+		{{ render_field(form.file_filter, hint="Supports wildcards and regex") }}
+		{{ render_submit_field(form.submit, tabindex=180) }}
+	</form>
+
+	<p class="mt-5">
+		For more information, see <a href="https://linux.die.net/man/1/zipgrep">ZipGrep's man page</a>.
+	</p>
+{% endblock %}

app/templates/zipgrep/view_results.html

@@ -0,0 +1,47 @@
+{% extends "base.html" %}
+
+{% block title %}
+{{ _("'%(query)s' - Search Package Releases", query=query) }}
+{% endblock %}
+
+{% block content %}
+	<a class="btn btn-secondary float-right" href="{{ url_for('zipgrep.zipgrep_search') }}">New Query</a>
+	<h1>{{ _("Search in Package Releases") }}</h1>
+	<h2>{{ query }}</h2>
+
+	<p class="text-muted">
+		Found in {{ matches | count }} packages.
+	</p>
+
+	<div class="list-group">
+		{% for match in matches %}
+			<div class="list-group-item">
+				<div class="row">
+					<div class="col-sm-2 text-muted">
+						<img
+							class="img-fluid"
+							src="{{ match.package.getThumbnailOrPlaceholder() }}" />
+
+						<div class="mt-2">
+							<a href="{{ match.package.getURL('packages.view') }}">
+								{{ match.package.title }}
+							</a>
+							by {{ match.package.author.display_name }}
+						</div>
+
+						<p class="mt-4">
+							{{ match.lines.split("\n") | select | list | count }} matches
+						</p>
+
+						<a class="mt-4 btn btn-secondary" href="{{ match.package.getDownloadRelease().getDownloadURL() }}">
+							Download
+						</a>
+					</div>
+					<div class="col-sm-10">
+						<pre style="max-height: 300px;" class="m-0">{{ match.lines }}</pre>
+					</div>
+				</div>
+			</div>
+		{% endfor %}
+	</div>
+{% endblock %}