feat: added retention_days deletion

app/models/app.py

@@ -4,4 +4,4 @@ from sqlalchemy import Column, Integer, String
 
 class Application(Base):
     name = Column(String(512), nullable=False, unique=True)
-    retention_days = Column(Integer, nullable=True, default=30)
+    retention_days = Column(Integer, nullable=True)

app/setup.py

@@ -1,12 +1,32 @@
 import os
+from datetime import datetime, timedelta
 
-from creyPY.fastapi.db.session import SQLALCHEMY_DATABASE_URL, name
+from apscheduler.schedulers.background import BackgroundScheduler
+from creyPY.fastapi.db.session import SQLALCHEMY_DATABASE_URL, get_db, name
+from sqlalchemy.orm import Session
 
 from alembic import command
 from alembic.config import Config
+from app.models.app import Application
+from app.models.entry import LogEntry
 from app.services.db.session import create_if_not_exists
 
 
+def delete_old_logs(sess: Session | None = None):
+    session = sess or next(get_db())
+
+    for app in session.query(Application).filter(Application.retention_days.isnot(None)):
+        cutoff = datetime.now() - timedelta(days=app.retention_days)
+        print(
+            f"Deleting logs older than {app.retention_days} days (cutoff: {cutoff}) for {app.name}",
+        )
+        session.query(LogEntry).filter(
+            LogEntry.application == app.id, LogEntry.created_at < cutoff
+        ).delete()
+
+    session.commit()
+
+
 def setup(db_name=name):
     # Create Database
     create_if_not_exists(db_name)
@@ -18,3 +38,17 @@ def setup(db_name=name):
         "script_location", os.path.join(os.path.dirname(os.path.dirname(__file__)), "alembic")
     )
     command.upgrade(config, "head")
+
+    # Start retention deletion
+    scheduler = BackgroundScheduler()
+    scheduler.add_job(
+        delete_old_logs,
+        "interval",
+        id="deletor",
+        days=1,
+        max_instances=1,
+        replace_existing=True,
+        next_run_time=datetime.now(),
+    )
+    scheduler.start()
+    print("Deletion scheduler started")

app/test_main.py

@@ -1,3 +1,6 @@
+import contextlib
+from datetime import datetime, timedelta
+
 from creyPY.fastapi.db.session import SQLALCHEMY_DATABASE_URL, get_db
 from creyPY.fastapi.models.base import Base
 from creyPY.fastapi.testing import GenericClient
@@ -5,25 +8,14 @@ from sqlalchemy import create_engine
 from sqlalchemy.orm import sessionmaker
 from sqlalchemy_utils import create_database, database_exists, drop_database
 
+from app.models.entry import LogEntry
 from app.services.auth import verify
-import contextlib
+from app.setup import delete_old_logs
+
 from .main import app
 
 CURRENT_USER = "api-key|testing"
-
-
-@contextlib.contextmanager
-def app_context(self, name: str = "Testing"):
-    app_id = self.create_app(name)
-    try:
-        yield app_id
-    finally:
-        self.destroy_app(app_id)
-
-
-@contextlib.contextmanager
-def log_examples(self):
-    LOG_EXAMPLES = [
+ENTRY_EXAMPLES = [
     {
         "l_type": "info",
         "t_type": "create",
@@ -60,8 +52,21 @@ def log_examples(self):
         "environment": "prod",
     },
 ]
+
+
+@contextlib.contextmanager
+def app_context(self, name: str = "Testing", retention_days: int | None = None):
+    app_id = self.create_app(name, retention_days)
+    try:
+        yield app_id
+    finally:
+        self.destroy_app(app_id)
+
+
+@contextlib.contextmanager
+def log_examples(self):
     with app_context(self) as app_id:
-        for entry in LOG_EXAMPLES:
+        for entry in ENTRY_EXAMPLES:
             self.log_message({"application": app_id, **entry})
         yield app_id
 
@@ -86,6 +91,7 @@ class TestAPI:
             global CURRENT_USER
             return CURRENT_USER
 
+        self.db_instance = get_db_test()
         app.dependency_overrides[get_db] = get_db_test
         app.dependency_overrides[verify] = get_test_sub
         self.c = GenericClient(app)
@@ -94,8 +100,8 @@ class TestAPI:
         drop_database(self.engine.url)
 
     # HELPERS
-    def create_app(self, name: str = "Testing"):
-        re = self.c.post("/app/", {"name": name})
+    def create_app(self, name: str = "Testing", retention_days: int | None = None):
+        re = self.c.post("/app/", {"name": name, "retention_days": retention_days})
         return re["id"]
 
     def destroy_app(self, app_id):
@@ -260,3 +266,29 @@ class TestAPI:
 
             re = self.c.get("/log/?application=" + str(app_id))
             assert re["total"] == 0
+
+    def test_retention_delete(self):
+        sess = next(self.db_instance)
+
+        with app_context(self, retention_days=2) as app_id:
+            for i in range(5):
+                sess.add(
+                    LogEntry(
+                        application=app_id,
+                        created_at=datetime.now() - timedelta(days=i),
+                        created_by_id=CURRENT_USER,
+                    )
+                )
+            sess.commit()
+
+            assert sess.query(LogEntry).count() == 5
+
+            re = self.c.get("/log/?application=" + str(app_id))
+            assert re["total"] == 5
+
+            delete_old_logs(sess)
+
+            assert sess.query(LogEntry).count() == 2
+
+            # delete all logs
+            re = self.c.delete("/log/?application=" + str(app_id), r_code=200)

requirements.txt

@@ -34,3 +34,6 @@ pluggy==1.5.0 # pytest
 pytest==8.3.4 # pytest
 
 fastapi-filters==0.2.9 # Filters
+
+APScheduler==3.11.0 # Scheduler for deletion
+tzlocal==5.2 # Scheduler for deletion