From e15a3c682fd784621bfd87ba11969352829fa5f4 Mon Sep 17 00:00:00 2001 From: rubenwardy Date: Wed, 9 Nov 2022 18:46:11 +0000 Subject: [PATCH] Statistics: Fix issue with missing days Fixes #397 --- app/blueprints/api/endpoints.py | 15 ++++++-- app/flatpages/help/api.md | 14 ++++--- app/logic/graphs.py | 59 +++++++++++++++++++++++++---- app/public/static/package_charts.js | 11 +++++- app/tests/unit/test_logic_graphs.py | 33 ++++++++++++++++ 5 files changed, 115 insertions(+), 17 deletions(-) create mode 100644 app/tests/unit/test_logic_graphs.py diff --git a/app/blueprints/api/endpoints.py b/app/blueprints/api/endpoints.py index 98822390..baa7a2d2 100644 --- a/app/blueprints/api/endpoints.py +++ b/app/blueprints/api/endpoints.py @@ -29,14 +29,13 @@ from app.models import Tag, PackageState, PackageType, Package, db, PackageRelea MinetestRelease, APIToken, PackageScreenshot, License, ContentWarning, User, PackageReview, Thread from app.querybuilder import QueryBuilder from app.utils import is_package_page, get_int_or_abort, url_set_query, abs_url, isYes +from app.logic.graphs import get_package_stats, get_package_stats_for_user from . import bp from .auth import is_api_authd from .support import error, api_create_vcs_release, api_create_zip_release, api_create_screenshot, \ api_order_screenshots, api_edit_package, api_set_cover_image from functools import wraps -from ...logic.graphs import flatten_data - def cors_allowed(f): @wraps(f) @@ -439,7 +438,7 @@ def list_all_reviews(): @is_package_page @cors_allowed def package_stats(package: Package): - return jsonify(flatten_data(package)) + return jsonify(get_package_stats(package)) @bp.route("/api/scores/") @@ -577,3 +576,13 @@ def all_deps(): }, "items": [format_pkg(pkg) for pkg in pagination.items], }) + + +@bp.route("/api/users//stats/") +@cors_allowed +def user_stats(username: str): + user = User.query.filter_by(username=username).first() + if user is None: + error(404, "User not found") + + return jsonify(get_package_stats_for_user(user)) diff --git a/app/flatpages/help/api.md b/app/flatpages/help/api.md index c804840d..d2b00177 100644 --- a/app/flatpages/help/api.md +++ b/app/flatpages/help/api.md @@ -110,14 +110,16 @@ Tokens can be attained by visiting [Settings > API Tokens](/user/tokens/). * Same as above. * GET `/api/packages///stats/` * Returns daily stats for package, or null if there is no data. + * Daily date is done based on the UTC timezone. * EXPERIMENTAL. This API may change without warning. * A table with the following keys: - * `dates`: list of dates in isoformat - * `platform_minetest`: list of integers per date - * `platform_other`: list of integers per date - * `reason_new`: list of integers per date - * `reason_dependency`: list of integers per date - * `reason_update`: list of integers per date + * `from`: start date, inclusive. Ex: 2022-10-22. + * `end`: end date, inclusive. Ex: 2022-11-05. + * `platform_minetest`: list of integers per day. + * `platform_other`: list of integers per day. + * `reason_new`: list of integers per day. + * `reason_dependency`: list of integers per day. + * `reason_update`: list of integers per day. You can download a package by building one of the two URLs: diff --git a/app/logic/graphs.py b/app/logic/graphs.py index 96af1ce1..99f3962f 100644 --- a/app/logic/graphs.py +++ b/app/logic/graphs.py @@ -1,17 +1,62 @@ -from app.models import Package, PackageDailyStats, db +from datetime import timedelta +from app.models import User, Package, PackageDailyStats, db +from sqlalchemy import func -def flatten_data(package: Package): - stats = package.daily_stats.order_by(db.asc(PackageDailyStats.date)).all() +def daterange(start_date, end_date): + for n in range(int((end_date - start_date).days) + 1): + yield start_date + timedelta(n) + + +keys = ["platform_minetest", "platform_other", "reason_new", + "reason_dependency", "reason_update"] + + +def _flatten_data(stats): if len(stats) == 0: return None + start_date = stats[0].date + end_date = stats[-1].date result = { - "dates": [stat.date.isoformat() for stat in stats], + "start": start_date.isoformat(), + "end": end_date.isoformat(), } - for key in ["platform_minetest", "platform_other", "reason_new", - "reason_dependency", "reason_update"]: - result[key] = [getattr(stat, key) for stat in stats] + for key in keys: + result[key] = [] + + i = 0 + for date in daterange(start_date, end_date): + stat = stats[i] + if stat.date == date: + for key in keys: + result[key].append(getattr(stat, key)) + + i += 1 + else: + for key in keys: + result[key].append(0) return result + + +def get_package_stats(package: Package): + stats = package.daily_stats.order_by(db.asc(PackageDailyStats.date)).all() + return _flatten_data(stats) + + +def get_package_stats_for_user(user: User): + stats = db.session \ + .query(PackageDailyStats.date, + func.sum(PackageDailyStats.platform_minetest).label("platform_minetest"), + func.sum(PackageDailyStats.platform_other).label("platform_other"), + func.sum(PackageDailyStats.reason_new).label("reason_new"), + func.sum(PackageDailyStats.reason_dependency).label("reason_dependency"), + func.sum(PackageDailyStats.reason_update).label("reason_update")) \ + .filter(PackageDailyStats.package.has(author_id=user.id)) \ + .order_by(db.asc(PackageDailyStats.date)) \ + .group_by(PackageDailyStats.date) \ + .all() + + return _flatten_data(stats) diff --git a/app/public/static/package_charts.js b/app/public/static/package_charts.js index 334445cf..c2d3faef 100644 --- a/app/public/static/package_charts.js +++ b/app/public/static/package_charts.js @@ -32,6 +32,7 @@ function sum(list) { const chartColorsBg = chartColors.map(color => `rgba(${hexToRgb(color.slice(1))}, 0.2)`); +const SECONDS_IN_A_DAY = 1000 * 3600 * 24; async function load_data() { const root = document.getElementById("stats-root"); @@ -46,6 +47,14 @@ async function load_data() { return; } + const startDate = new Date(json.start); + const endDate = new Date(json.end); + const numberOfDays = Math.round((endDate.valueOf() - startDate.valueOf()) / SECONDS_IN_A_DAY) + 1; + const dates = [...Array(numberOfDays)].map((_, i) => { + const date = new Date(startDate.valueOf() + i*SECONDS_IN_A_DAY); + return date.toISOString().split('T')[0]; + }); + const total7 = sum(json.platform_minetest.slice(-7)) + sum(json.platform_other.slice(-7)); document.getElementById("downloads_total7d").textContent = total7; document.getElementById("downloads_avg7d").textContent = (total7 / 7).toFixed(0); @@ -66,7 +75,7 @@ async function load_data() { root.style.display = "block"; function getData(list) { - return list.map((value, i) => ({ x: json.dates[i], y: value })); + return list.map((value, i) => ({ x: dates[i], y: value })); } { diff --git a/app/tests/unit/test_logic_graphs.py b/app/tests/unit/test_logic_graphs.py new file mode 100644 index 00000000..a8620786 --- /dev/null +++ b/app/tests/unit/test_logic_graphs.py @@ -0,0 +1,33 @@ +import datetime + +from app.logic.graphs import _flatten_data + + +class DailyStat: + date: datetime.date + platform_minetest: int + platform_other: int + reason_new: int + reason_dependency: int + reason_update: int + + def __init__(self, date: str, x: int): + self.date = datetime.date.fromisoformat(date) + self.platform_minetest = x + self.platform_other = 0 + self.reason_new = 0 + self.reason_dependency = 0 + self.reason_update = 0 + + +def test_flatten_data(): + res = _flatten_data([ + DailyStat("2022-03-28", 3), + DailyStat("2022-03-29", 10), + DailyStat("2022-04-01", 5), + DailyStat("2022-04-02", 1) + ]) + + assert res["start"] == datetime.date.fromisoformat("2022-03-28") + assert res["end"] == datetime.date.fromisoformat("2022-04-02") + assert res["platform_minetest"] == [3, 10, 0, 0, 5, 1]