v0.22.1 (#339)

* filesystem huey

* bump servestatic

Author: Archmonger

VERSION

@@ -1 +1 @@
-0.22.0
+0.22.1

conreq/__init__.py

@@ -0,0 +1,39 @@
+# Monkey patch Huey's filelock implementation to add Windows support.
+import os
+import sys
+
+
+class FileLock:
+    """Operating system agnostic file lock implementation using os.open and os.close."""
+
+    def __init__(self, filename):
+        self.filename = filename
+        self.fd = None
+
+        dirname = os.path.dirname(filename)
+        if not os.path.exists(dirname):
+            os.makedirs(dirname)
+        elif os.path.exists(self.filename):
+            os.unlink(self.filename)
+
+    def acquire(self):
+        flags = os.O_CREAT | os.O_TRUNC | os.O_RDWR
+        self.fd = os.open(self.filename, flags)
+
+    def release(self):
+        if self.fd is not None:
+            fd, self.fd = self.fd, None
+            os.close(fd)
+
+    def __enter__(self):
+        self.acquire()
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        self.release()
+
+
+if sys.platform == "win32":
+    import huey.utils
+
+    huey.utils.FileLock = FileLock

conreq/core/base/management/commands/preconfig_conreq.py

@@ -13,7 +13,6 @@
 BASE_DIR = getattr(settings, "BASE_DIR")
 DATA_DIR = getattr(settings, "DATA_DIR")
 DATABASES = getattr(settings, "DATABASES")
-HUEY_FILENAME = getattr(settings, "HUEY_FILENAME")
 
 
 class Command(BaseCommand):
@@ -34,12 +33,6 @@ def handle(self, *args, **options):
             database = DATABASES["default"]["NAME"]
             self.setup_sqlite_database(database, "Conreq", uid, gid, no_perms)
 
-        # Background task database
-        if HUEY_FILENAME:
-            self.setup_sqlite_database(
-                HUEY_FILENAME, "Background Task", uid, gid, no_perms
-            )
-
         if DEBUG:
             # Migrate silk due to their wonky dev choices
             call_command("makemigrations", "silk")

conreq/core/base/management/commands/run_conreq.py

@@ -16,7 +16,6 @@
 
 UVICORN_CONFIG = os.path.join(getattr(settings, "DATA_DIR"), "uvicorn.env")
 DEBUG = get_debug()
-HUEY_FILENAME = getattr(settings, "HUEY_FILENAME")
 ACCESS_LOG_FILE = getattr(settings, "ACCESS_LOG_FILE")
 
 _logger = getLogger(__name__)

conreq/core/base/static/js/events_click.js

@@ -133,7 +133,6 @@ var quick_request_click_event = async function () {
 			// Request the content
 			post_json(btn.data("request-url"), params, function () {
 				requested_toast_message();
-				console.log(btn);
 				btn.remove();
 				ongoing_request = null;
 			}).fail(async function () {

conreq/core/base/tasks.py

@@ -1,37 +1,10 @@
-import sqlite3
-
-from django.conf import settings
 from django.db import connection
 from huey import crontab
 from huey.contrib.djhuey import db_periodic_task
 
 from conreq.utils.environment import get_database_type
 
 DB_ENGINE = get_database_type()
-HUEY_FILENAME = getattr(settings, "HUEY_FILENAME")
-
-
-@db_periodic_task(crontab(minute="0", hour="0", strict=True), expires=120)
-def huey_db_maintenance():
-    with sqlite3.connect(HUEY_FILENAME) as cursor:
-        cursor.execute(
-            # Only keep the 1000 latest tasks
-            """DELETE FROM task
-            WHERE id NOT IN (
-            SELECT id
-            FROM (
-                SELECT id
-                FROM task
-                ORDER BY id DESC
-                LIMIT 1000
-            ) foo
-            );
-            """
-        )
-    with sqlite3.connect(HUEY_FILENAME) as cursor:
-        cursor.execute("PRAGMA optimize;")
-        cursor.execute("VACUUM;")
-        cursor.execute("REINDEX;")
 
 
 if DB_ENGINE == "SQLITE3":

conreq/settings.py

@@ -90,25 +90,15 @@
     "css": ["compressor.filters.cssmin.rCSSMinFilter"],
     "js": ["compressor.filters.jsmin.JSMinFilter"],
 }
-HUEY_FILENAME = os.path.join(DATA_DIR, "bg_tasks.sqlite3")
 HUEY = {
-    "name": "huey",  # DB name for huey.
-    "huey_class": "huey.SqliteHuey",  # Huey implementation to use.
-    "filename": HUEY_FILENAME,  # Sqlite filename
-    "results": True,  # Whether to return values of tasks.
-    "store_none": False,  # Whether to store results of tasks that return None.
-    "immediate": False,  # If True, run tasks synchronously.
-    "strict_fifo": True,  # Utilize Sqlite AUTOINCREMENT to have unique task IDs
-    "timeout": 10,  # Seconds to wait when reading from the DB.
-    "connection": {
-        "isolation_level": "IMMEDIATE",  # Use immediate transactions to allow sqlite to respect `timeout`.
-        "cached_statements": 2000,  # Number of pages to keep in memory.
-    },
+    "name": "huey",
+    "huey_class": "huey.FileHuey",
+    "path": os.path.join(DATA_DIR, "tasks"),
+    "immediate": False,
+    "use_thread_lock": True,
     "consumer": {
         "workers": os.cpu_count() or 8,  # Number of worker processes/threads.
-        "worker_type": "thread",  # "thread" or "process"
         "initial_delay": 0.25,  # Smallest polling interval
-        "check_worker_health": True,  # Whether to monitor worker health.
     },
 }
 

requirements/main.txt

@@ -19,7 +19,7 @@ titlecase==2.4.1
 tmdbsimple==2.9.1
 Twisted[tls,http2]==25.5.0
 tzlocal==5.3.1
-servestatic[brotli]==3.1.0
+servestatic[brotli]==4.1.0
 uvicorn[standard]==0.38.0
 attrs
 cffi