contentdb/utils/import_nginx_logs.py
2022-11-06 16:54:06 +00:00

150 lines
3.8 KiB
Python
Executable File

# ContentDB
# Copyright (C) 2022 rubenwardy
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <https://www.gnu.org/licenses/>.
import datetime
import inspect
import os
import re
import sys
import gzip
import user_agents
from urllib.parse import urlparse, parse_qs, unquote
if not "FLASK_CONFIG" in os.environ:
os.environ["FLASK_CONFIG"] = "../config.cfg"
logs_dir = sys.argv[1].strip()
if not os.path.isdir(logs_dir):
sys.exit(1)
# Allow finding the `app` module
currentdir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe())))
parentdir = os.path.dirname(currentdir)
sys.path.insert(0,parentdir)
from app.models import db, Package, PackageDailyStats
url_re = re.compile(r"\/packages\/([^\/]+)\/([^\/]+)\/releases\/[0-9]+\/download\/[^ ]*")
dt_re = re.compile(r"\[(\d+\/\w+\/[\d: +-]+)\]")
ua_re = re.compile(r"\"([^\"]+)\"$")
row_lookup = {}
package_id_by_key = {}
for package in Package.query.all():
package_id_by_key[f"{package.author.username}/{package.name}".lower()] = package.id
log_files = [f for f in os.listdir(logs_dir) if os.path.isfile(os.path.join(logs_dir, f))]
ua_is_bot = {}
def my_open(path):
if path.endswith(".gz"):
return gzip.open(path, "rt")
else:
return open(path, "r")
for log_file in log_files:
print(f"Importing from {log_file}")
with my_open(os.path.join(logs_dir, log_file)) as infile:
line_no = 1
for line in infile:
if "/download/" not in line:
continue
line_no += 1
url_match = url_re.search(line)
if url_match is None:
continue
url = url_match.group(0)
author = unquote(url_match.group(1))
name = url_match.group(2)
parsed_url = urlparse(url)
reason = parse_qs(parsed_url.query).get("reason")
if reason:
reason = reason[0]
dt_match = dt_re.search(line)
assert dt_match
dt = datetime.datetime.strptime(dt_match.group(1), "%d/%b/%Y:%H:%M:%S %z")
dt = datetime.datetime.utcfromtimestamp(dt.timestamp())
date = dt.date()
if date >= datetime.date(2022, 11, 6):
continue
# print(line)
ua_match = ua_re.search(line)
if ua_match is None:
print("No UA: " + line)
continue
ua = ua_match.group(1)
is_bot = ua_is_bot.get(ua)
if is_bot:
continue
if is_bot is None:
user_agent = user_agents.parse(ua)
ua_is_bot[ua] = user_agent.is_bot
if user_agent.is_bot:
continue
package_key = f"{author}/{name}".lower()
package_id = package_id_by_key.get(package_key)
if package_id is None:
print(f"Package not found: {package_key}")
continue
key = f"{date.isoformat()}/{package_id}"
# print(author, name, reason, ua, date, key)
row = row_lookup.get(key)
if not row:
row = PackageDailyStats()
row.date = date
row.package_id = package_id
row.platform_minetest = 0
row.platform_other = 0
row.reason_new = 0
row.reason_dependency = 0
row.reason_update = 0
db.session.add(row)
row_lookup[key] = row
if ua.startswith("Minetest/"):
row.platform_minetest += 1
else:
row.platform_other += 1
if reason == "new":
row.reason_new += 1
elif reason == "dependency":
row.reason_dependency += 1
elif reason == "update":
row.reason_update += 1
# if line_no > 1000:
# break
db.session.commit()