Mirrorlandia_minetest/contentdb
1
0
Fork 0
mirror of https://github.com/minetest/contentdb.git synced 2024-12-22 22:12:24 +01:00
Code Issues Packages Projects Releases Wiki Activity

Add fulltext search support

Browse Source
This commit is contained in:
rubenwardy 2019-01-29 03:00:01 +00:00
parent d36138d5e1
commit 2586a11bcf
14 changed files with 334 additions and 34 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

42
app/models.py
View File

@ -15,18 +15,29 @@
# along with this program. If not, see <https://www.gnu.org/licenses/>.
from flask import Flask, url_for
from flask_sqlalchemy import SQLAlchemy
from flask_migrate import Migrate
from urllib.parse import urlparse
from app import app, gravatar
from sqlalchemy.orm import validates
from flask_user import login_required, UserManager, UserMixin, SQLAlchemyAdapter
import enum, datetime
from app import app, gravatar
from urllib.parse import urlparse
from flask import Flask, url_for
from flask_sqlalchemy import SQLAlchemy, BaseQuery
from flask_migrate import Migrate
from flask_user import login_required, UserManager, UserMixin, SQLAlchemyAdapter
from sqlalchemy.orm import validates
from sqlalchemy_searchable import SearchQueryMixin
from sqlalchemy_utils.types import TSVectorType
from sqlalchemy_searchable import make_searchable
# Initialise database
db = SQLAlchemy(app)
migrate = Migrate(app, db)
make_searchable(db.metadata)
class ArticleQuery(BaseQuery, SearchQueryMixin):
pass
class UserRank(enum.Enum):
@ -246,7 +257,7 @@ class PackageType(enum.Enum):
class PackagePropertyKey(enum.Enum):
name = "Name"
title = "Title"
shortDesc = "Short Description"
short_desc = "Short Description"
desc = "Description"
type = "Type"
license = "License"
@ -343,19 +354,22 @@ class Dependency(db.Model):
return retval
class Package(db.Model):
query_class = ArticleQuery
id = db.Column(db.Integer, primary_key=True)
# Basic details
author_id = db.Column(db.Integer, db.ForeignKey("user.id"))
name = db.Column(db.String(100), nullable=False)
title = db.Column(db.String(100), nullable=False)
shortDesc = db.Column(db.String(200), nullable=False)
desc = db.Column(db.Text, nullable=True)
title = db.Column(db.Unicode(100), nullable=False)
short_desc = db.Column(db.Unicode(200), nullable=False)
desc = db.Column(db.UnicodeText, nullable=True)
type = db.Column(db.Enum(PackageType))
created_at = db.Column(db.DateTime, nullable=False, default=datetime.datetime.utcnow)
search_vector = db.Column(TSVectorType("title", "short_desc", "desc"))
license_id = db.Column(db.Integer, db.ForeignKey("license.id"), nullable=False, default=1)
license = db.relationship("License", foreign_keys=[license_id])
media_license_id = db.Column(db.Integer, db.ForeignKey("license.id"), nullable=False, default=1)
@ -409,7 +423,7 @@ class Package(db.Model):
"name": self.name,
"title": self.title,
"author": self.author.display_name,
"short_description": self.shortDesc,
"short_description": self.short_desc,
"type": self.type.toName(),
"release": self.getDownloadRelease(protonum).id if self.getDownloadRelease(protonum) is not None else None,
"thumbnail": (base_url + tnurl) if tnurl is not None else None,
@ -422,7 +436,7 @@ class Package(db.Model):
"author": self.author.display_name,
"name": self.name,
"title": self.title,
"short_description": self.shortDesc,
"short_description": self.short_desc,
"desc": self.desc,
"type": self.type.toName(),
"created_at": self.created_at,

4
app/public/static/package_create.js
View File

@ -35,10 +35,10 @@ $(function() {
setField("#repo", result.repo || repoURL);
setField("#issueTracker", result.issueTracker);
setField("#desc", result.description);
setField("#shortDesc", result.short_description);
setField("#short_desc", result.short_description);
setField("#harddep_str", result.depends);
setField("#softdep_str", result.optional_depends);
setField("#shortDesc", result.short_description);
setField("#short_desc", result.short_description);
setField("#forums", result.forumId);
if (result.type && result.type.length > 2) {
$("#type").val(result.type);

2
app/public/static/package_edit.js
View File

@ -41,7 +41,7 @@ $(function() {
It's obvious that this adds something to Minetest,
there's no need to use phrases such as \"adds X to the game\".`
$("#shortDesc").on("change paste keyup", function() {
$("#short_desc").on("change paste keyup", function() {
var val = $(this).val().toLowerCase();
if (val.indexOf("minetest") >= 0 || val.indexOf("mod") >= 0 ||
val.indexOf("modpack") >= 0 || val.indexOf("mod pack") >= 0) {

2
app/querybuilder.py
View File

@ -40,7 +40,7 @@ class QueryBuilder:
query = query.filter(Package.type.in_(self.types))
if self.search:
query = query.filter(Package.title.ilike('%' + self.search + '%'))
query = query.search(self.search)
if self.random:
query = query.order_by(func.random())

2
app/templates/macros/packagegridtile.html
View File

@ -12,7 +12,7 @@
</h3>
<p>
{{ package.shortDesc }}
{{ package.short_desc }}
</p>

2
app/templates/packages/create_edit.html
View File

@ -49,7 +49,7 @@
{{ render_field(form.title, class_="pkg_meta col-sm-7") }}
{{ render_field(form.name, class_="pkg_meta col-sm-3") }}
</div>
{{ render_field(form.shortDesc, class_="pkg_meta") }}
{{ render_field(form.short_desc, class_="pkg_meta") }}
{{ render_multiselect_field(form.tags, class_="pkg_meta") }}
<div class="pkg_meta row">
{{ render_field(form.license, class_="not_txp col-sm-6") }}

2
app/templates/packages/editrequest_create_edit.html
View File

@ -17,7 +17,7 @@
{{ render_field(form.type) }}
{{ render_field(form.name) }}
{{ render_field(form.title) }}
{{ render_field(form.shortDesc) }}
{{ render_field(form.short_desc) }}
{{ render_field(form.desc) }}
{{ render_multiselect_field(form.tags) }}

2
app/templates/packages/view.html
View File

@ -19,7 +19,7 @@
</h1>
<p class="lead">
{{ package.shortDesc }}
{{ package.short_desc }}
</p>
<div class="row" style="margin-top: 2rem;">

2
app/views/packages/packages.py
View File

@ -171,7 +171,7 @@ def package_download_page(package):
class PackageForm(FlaskForm):
name = StringField("Name (Technical)", [InputRequired(), Length(1, 20), Regexp("^[a-z0-9_]", 0, "Lower case letters (a-z), digits (0-9), and underscores (_) only")])
title = StringField("Title (Human-readable)", [InputRequired(), Length(3, 50)])
shortDesc = StringField("Short Description (Plaintext)", [InputRequired(), Length(1,200)])
short_desc = StringField("Short Description (Plaintext)", [InputRequired(), Length(1,200)])
desc = TextAreaField("Long Description (Markdown)", [Optional(), Length(0,10000)])
type = SelectField("Type", [InputRequired()], choices=PackageType.choices(), coerce=PackageType.coerce, default=PackageType.MOD)
license = QuerySelectField("License", [InputRequired()], query_factory=lambda: License.query.order_by(db.asc(License.name)), get_pk=lambda a: a.id, get_label=lambda a: a.name)

36
migrations/versions/2f3c3597c78d_.py Normal file
View File

@ -0,0 +1,36 @@
"""empty message
Revision ID: 2f3c3597c78d
Revises: 9ec17b558413
Create Date: 2019-01-29 02:43:08.865695
"""
from alembic import op
import sqlalchemy as sa
from sqlalchemy.dialects import postgresql
from sqlalchemy_utils.types import TSVectorType
from sqlalchemy_searchable import sync_trigger
# revision identifiers, used by Alembic.
revision = '2f3c3597c78d'
down_revision = '9ec17b558413'
branch_labels = None
depends_on = None
def upgrade():
# ### commands auto generated by Alembic - please adjust! ###
op.alter_column('package', 'short_desc', nullable=False, new_column_name='short_desc')
op.add_column('package', sa.Column('search_vector', TSVectorType("title", "short_desc", "desc"), nullable=True))
op.create_index('ix_package_search_vector', 'package', ['search_vector'], unique=False, postgresql_using='gin')
conn = op.get_bind()
sync_trigger(conn, 'package', 'search_vector', ["title", "short_desc", "desc"])
# ### end Alembic commands ###
def downgrade():
# ### commands auto generated by Alembic - please adjust! ###
op.drop_index('ix_package_search_vector', table_name='package')
op.drop_column('package', 'search_vector')
# ### end Alembic commands ###

249
migrations/versions/7ff57806ffd5_.py Normal file
View File

@ -0,0 +1,249 @@
"""empty message
Revision ID: 7ff57806ffd5
Revises: 2f3c3597c78d
Create Date: 2019-01-29 02:57:50.279918
"""
from alembic import op
import sqlalchemy as sa
from sqlalchemy.dialects import postgresql
# revision identifiers, used by Alembic.
revision = '7ff57806ffd5'
down_revision = '2f3c3597c78d'
branch_labels = None
depends_on = None
def upgrade():
# ### commands auto generated by Alembic - please adjust! ###
op.execute("""
DROP TYPE IF EXISTS tsq_state CASCADE;
CREATE TYPE tsq_state AS (
search_query text,
parentheses_stack int,
skip_for int,
current_token text,
current_index int,
current_char text,
previous_char text,
tokens text[]
);
CREATE OR REPLACE FUNCTION tsq_append_current_token(state tsq_state)
RETURNS tsq_state AS $$
BEGIN
IF state.current_token != '' THEN
state.tokens := array_append(state.tokens, state.current_token);
state.current_token := '';
END IF;
RETURN state;
END;
$$ LANGUAGE plpgsql IMMUTABLE;
CREATE OR REPLACE FUNCTION tsq_tokenize_character(state tsq_state)
RETURNS tsq_state AS $$
BEGIN
IF state.current_char = '(' THEN
state.tokens := array_append(state.tokens, '(');
state.parentheses_stack := state.parentheses_stack + 1;
state := tsq_append_current_token(state);
ELSIF state.current_char = ')' THEN
IF (state.parentheses_stack > 0 AND state.current_token != '') THEN
state := tsq_append_current_token(state);
state.tokens := array_append(state.tokens, ')');
state.parentheses_stack := state.parentheses_stack - 1;
END IF;
ELSIF state.current_char = '"' THEN
state.skip_for := position('"' IN substring(
state.search_query FROM state.current_index + 1
));
IF state.skip_for > 1 THEN
state.tokens = array_append(
state.tokens,
substring(
state.search_query
FROM state.current_index FOR state.skip_for + 1
)
);
ELSIF state.skip_for = 0 THEN
state.current_token := state.current_token || state.current_char;
END IF;
ELSIF (
state.current_char = '-' AND
(state.current_index = 1 OR state.previous_char = ' ')
) THEN
state.tokens := array_append(state.tokens, '-');
ELSIF state.current_char = ' ' THEN
state := tsq_append_current_token(state);
IF substring(
state.search_query FROM state.current_index FOR 4
) = ' or ' THEN
state.skip_for := 2;
-- remove duplicate OR tokens
IF state.tokens[array_length(state.tokens, 1)] != ' | ' THEN
state.tokens := array_append(state.tokens, ' | ');
END IF;
END IF;
ELSE
state.current_token = state.current_token || state.current_char;
END IF;
RETURN state;
END;
$$ LANGUAGE plpgsql IMMUTABLE;
CREATE OR REPLACE FUNCTION tsq_tokenize(search_query text) RETURNS text[] AS $$
DECLARE
state tsq_state;
BEGIN
SELECT
search_query::text AS search_query,
0::int AS parentheses_stack,
0 AS skip_for,
''::text AS current_token,
0 AS current_index,
''::text AS current_char,
''::text AS previous_char,
'{}'::text[] AS tokens
INTO state;
state.search_query := lower(trim(
regexp_replace(search_query, '""+', '""', 'g')
));
FOR state.current_index IN (
SELECT generate_series(1, length(state.search_query))
) LOOP
state.current_char := substring(
search_query FROM state.current_index FOR 1
);
IF state.skip_for > 0 THEN
state.skip_for := state.skip_for - 1;
CONTINUE;
END IF;
state := tsq_tokenize_character(state);
state.previous_char := state.current_char;
END LOOP;
state := tsq_append_current_token(state);
state.tokens := array_nremove(state.tokens, '(', -state.parentheses_stack);
RETURN state.tokens;
END;
$$ LANGUAGE plpgsql IMMUTABLE;
-- Processes an array of text search tokens and returns a tsquery
CREATE OR REPLACE FUNCTION tsq_process_tokens(config regconfig, tokens text[])
RETURNS tsquery AS $$
DECLARE
result_query text;
previous_value text;
value text;
BEGIN
result_query := '';
FOREACH value IN ARRAY tokens LOOP
IF value = '"' THEN
CONTINUE;
END IF;
IF left(value, 1) = '"' AND right(value, 1) = '"' THEN
value := phraseto_tsquery(config, value);
ELSIF value NOT IN ('(', ' | ', ')', '-') THEN
value := quote_literal(value) || ':*';
END IF;
IF previous_value = '-' THEN
IF value = '(' THEN
value := '!' || value;
ELSE
value := '!(' || value || ')';
END IF;
END IF;
SELECT
CASE
WHEN result_query = '' THEN value
WHEN (
previous_value IN ('!(', '(', ' | ') OR
value IN (')', ' | ')
) THEN result_query || value
ELSE result_query || ' & ' || value
END
INTO result_query;
previous_value := value;
END LOOP;
RETURN to_tsquery(config, result_query);
END;
$$ LANGUAGE plpgsql IMMUTABLE;
CREATE OR REPLACE FUNCTION tsq_process_tokens(tokens text[])
RETURNS tsquery AS $$
SELECT tsq_process_tokens(get_current_ts_config(), tokens);
$$ LANGUAGE SQL IMMUTABLE;
CREATE OR REPLACE FUNCTION tsq_parse(config regconfig, search_query text)
RETURNS tsquery AS $$
SELECT tsq_process_tokens(config, tsq_tokenize(search_query));
$$ LANGUAGE SQL IMMUTABLE;
CREATE OR REPLACE FUNCTION tsq_parse(config text, search_query text)
RETURNS tsquery AS $$
SELECT tsq_parse(config::regconfig, search_query);
$$ LANGUAGE SQL IMMUTABLE;
CREATE OR REPLACE FUNCTION tsq_parse(search_query text) RETURNS tsquery AS $$
SELECT tsq_parse(get_current_ts_config(), search_query);
$$ LANGUAGE SQL IMMUTABLE;
-- remove first N elements equal to the given value from the array (array
-- must be one-dimensional)
--
-- If negative value is given as the third argument the removal of elements
-- starts from the last array element.
CREATE OR REPLACE FUNCTION array_nremove(anyarray, anyelement, int)
RETURNS ANYARRAY AS $$
WITH replaced_positions AS (
SELECT UNNEST(
CASE
WHEN $2 IS NULL THEN
'{}'::int[]
WHEN $3 > 0 THEN
(array_positions($1, $2))[1:$3]
WHEN $3 < 0 THEN
(array_positions($1, $2))[
(cardinality(array_positions($1, $2)) + $3 + 1):
]
ELSE
'{}'::int[]
END
) AS position
)
SELECT COALESCE((
SELECT array_agg(value)
FROM unnest($1) WITH ORDINALITY AS t(value, index)
WHERE index NOT IN (SELECT position FROM replaced_positions)
), $1[1:0]);
$$ LANGUAGE SQL IMMUTABLE;
""")
# ### end Alembic commands ###
def downgrade():
# ### commands auto generated by Alembic - please adjust! ###
pass
# ### end Alembic commands ###

4
migrations/versions/83622276d439_.py
View File

@ -66,7 +66,7 @@ def upgrade():
sa.Column('author_id', sa.Integer(), nullable=True),
sa.Column('name', sa.String(length=100), nullable=False),
sa.Column('title', sa.String(length=100), nullable=False),
sa.Column('shortDesc', sa.String(length=200), nullable=False),
sa.Column('short_desc', sa.String(length=200), nullable=False),
sa.Column('desc', sa.Text(), nullable=True),
sa.Column('type', sa.Enum('MOD', 'GAME', 'TXP', name='packagetype'), nullable=True),
sa.Column('license_id', sa.Integer(), nullable=True),
@ -141,7 +141,7 @@ def upgrade():
op.create_table('edit_request_change',
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('request_id', sa.Integer(), nullable=True),
sa.Column('key', sa.Enum('name', 'title', 'shortDesc', 'desc', 'type', 'license', 'tags', 'repo', 'website', 'issueTracker', 'forums', name='packagepropertykey'), nullable=False),
sa.Column('key', sa.Enum('name', 'title', 'short_desc', 'desc', 'type', 'license', 'tags', 'repo', 'website', 'issueTracker', 'forums', name='packagepropertykey'), nullable=False),
sa.Column('oldValue', sa.Text(), nullable=True),
sa.Column('newValue', sa.Text(), nullable=True),
sa.ForeignKeyConstraint(['request_id'], ['edit_request.id'], ),

1
requirements.txt
View File

@ -8,6 +8,7 @@ Flask-Migrate~=2.3
Flask-SQLAlchemy~=2.3
Flask-User~=0.6
GitHub-Flask~=3.2
SQLAlchemy-Searchable==1.0.3
beautifulsoup4~=4.6
celery~=4.2

18
setup.py
View File

@ -55,7 +55,7 @@ def defineDummyData(licenses, tags, ruben):
mod.repo = "https://github.com/ezhh/other_worlds"
mod.issueTracker = "https://github.com/ezhh/other_worlds/issues"
mod.forums = 16015
mod.shortDesc = "The content library should not be used yet as it is still in alpha"
mod.short_desc = "The content library should not be used yet as it is still in alpha"
mod.desc = "This is the long desc"
db.session.add(mod)
@ -77,7 +77,7 @@ def defineDummyData(licenses, tags, ruben):
mod1.repo = "https://github.com/rubenwardy/awards"
mod1.issueTracker = "https://github.com/rubenwardy/awards/issues"
mod1.forums = 4870
mod1.shortDesc = "Adds achievements and an API to register new ones."
mod1.short_desc = "Adds achievements and an API to register new ones."
mod1.desc = """
Majority of awards are back ported from Calinou's old fork in Carbone, under same license.
@ -112,7 +112,7 @@ awards.register_achievement("award_mesefind",{
mod2.repo = "https://github.com/minetest-mods/mesecons/"
mod2.issueTracker = "https://github.com/minetest-mods/mesecons/issues"
mod2.forums = 628
mod2.shortDesc = "Mesecons adds everything digital, from all kinds of sensors, switches, solar panels, detectors, pistons, lamps, sound blocks to advanced digital circuitry like logic gates and programmable blocks."
mod2.short_desc = "Mesecons adds everything digital, from all kinds of sensors, switches, solar panels, detectors, pistons, lamps, sound blocks to advanced digital circuitry like logic gates and programmable blocks."
mod2.desc = """
########################################################################
## __ __ _____ _____ _____ _____ _____ _ _ _____ ##
@ -210,7 +210,7 @@ No warranty is provided, express or implied, for any part of the project.
mod.repo = "https://github.com/ezhh/handholds"
mod.issueTracker = "https://github.com/ezhh/handholds/issues"
mod.forums = 17069
mod.shortDesc = "Adds hand holds and climbing thingies"
mod.short_desc = "Adds hand holds and climbing thingies"
mod.desc = "This is the long desc"
db.session.add(mod)
@ -233,7 +233,7 @@ No warranty is provided, express or implied, for any part of the project.
mod.repo = "https://github.com/ezhh/other_worlds"
mod.issueTracker = "https://github.com/ezhh/other_worlds/issues"
mod.forums = 16015
mod.shortDesc = "Adds space with asteroids and comets"
mod.short_desc = "Adds space with asteroids and comets"
mod.desc = "This is the long desc"
db.session.add(mod)
@ -248,7 +248,7 @@ No warranty is provided, express or implied, for any part of the project.
mod.repo = "https://github.com/rubenwardy/food/"
mod.issueTracker = "https://github.com/rubenwardy/food/issues/"
mod.forums = 2960
mod.shortDesc = "Adds lots of food and an API to manage ingredients"
mod.short_desc = "Adds lots of food and an API to manage ingredients"
mod.desc = "This is the long desc"
food = mod
db.session.add(mod)
@ -264,7 +264,7 @@ No warranty is provided, express or implied, for any part of the project.
mod.repo = "https://github.com/rubenwardy/food_sweet/"
mod.issueTracker = "https://github.com/rubenwardy/food_sweet/issues/"
mod.forums = 9039
mod.shortDesc = "Adds sweet food"
mod.short_desc = "Adds sweet food"
mod.desc = "This is the long desc"
food_sweet = mod
db.session.add(mod)
@ -282,7 +282,7 @@ No warranty is provided, express or implied, for any part of the project.
game1.repo = "https://github.com/rubenwardy/capturetheflag"
game1.issueTracker = "https://github.com/rubenwardy/capturetheflag/issues"
game1.forums = 12835
game1.shortDesc = "Two teams battle to snatch and return the enemy's flag, before the enemy takes their own!"
game1.short_desc = "Two teams battle to snatch and return the enemy's flag, before the enemy takes their own!"
game1.desc = """
As seen on the Capture the Flag server (minetest.rubenwardy.com:30000)
@ -307,7 +307,7 @@ Uses the CTF PvP Engine.
mod.type = PackageType.TXP
mod.author = ruben
mod.forums = 14132
mod.shortDesc = "This is an update of the original PixelBOX texture pack by the brillant artist Gambit"
mod.short_desc = "This is an update of the original PixelBOX texture pack by the brillant artist Gambit"
mod.desc = "This is the long desc"
db.session.add(mod)