From 4e7f352a154fd059b52e5df4b8de3d62a8ff9497 Mon Sep 17 00:00:00 2001 From: Conrad Date: Mon, 20 Jan 2025 13:04:18 +0100 Subject: [PATCH] feat: added retention_days deletion --- app/models/app.py | 2 +- app/setup.py | 36 +++++++++++++- app/test_main.py | 118 +++++++++++++++++++++++++++++----------------- requirements.txt | 3 ++ 4 files changed, 114 insertions(+), 45 deletions(-) diff --git a/app/models/app.py b/app/models/app.py index ac8d2fa..310349e 100644 --- a/app/models/app.py +++ b/app/models/app.py @@ -4,4 +4,4 @@ from sqlalchemy import Column, Integer, String class Application(Base): name = Column(String(512), nullable=False, unique=True) - retention_days = Column(Integer, nullable=True, default=30) + retention_days = Column(Integer, nullable=True) diff --git a/app/setup.py b/app/setup.py index 959368f..727ca46 100644 --- a/app/setup.py +++ b/app/setup.py @@ -1,12 +1,32 @@ import os +from datetime import datetime, timedelta -from creyPY.fastapi.db.session import SQLALCHEMY_DATABASE_URL, name +from apscheduler.schedulers.background import BackgroundScheduler +from creyPY.fastapi.db.session import SQLALCHEMY_DATABASE_URL, get_db, name +from sqlalchemy.orm import Session from alembic import command from alembic.config import Config +from app.models.app import Application +from app.models.entry import LogEntry from app.services.db.session import create_if_not_exists +def delete_old_logs(sess: Session | None = None): + session = sess or next(get_db()) + + for app in session.query(Application).filter(Application.retention_days.isnot(None)): + cutoff = datetime.now() - timedelta(days=app.retention_days) + print( + f"Deleting logs older than {app.retention_days} days (cutoff: {cutoff}) for {app.name}", + ) + session.query(LogEntry).filter( + LogEntry.application == app.id, LogEntry.created_at < cutoff + ).delete() + + session.commit() + + def setup(db_name=name): # Create Database create_if_not_exists(db_name) @@ -18,3 +38,17 @@ def setup(db_name=name): "script_location", os.path.join(os.path.dirname(os.path.dirname(__file__)), "alembic") ) command.upgrade(config, "head") + + # Start retention deletion + scheduler = BackgroundScheduler() + scheduler.add_job( + delete_old_logs, + "interval", + id="deletor", + days=1, + max_instances=1, + replace_existing=True, + next_run_time=datetime.now(), + ) + scheduler.start() + print("Deletion scheduler started") diff --git a/app/test_main.py b/app/test_main.py index 45d58fd..62708fb 100644 --- a/app/test_main.py +++ b/app/test_main.py @@ -1,3 +1,6 @@ +import contextlib +from datetime import datetime, timedelta + from creyPY.fastapi.db.session import SQLALCHEMY_DATABASE_URL, get_db from creyPY.fastapi.models.base import Base from creyPY.fastapi.testing import GenericClient @@ -5,16 +8,55 @@ from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker from sqlalchemy_utils import create_database, database_exists, drop_database +from app.models.entry import LogEntry from app.services.auth import verify -import contextlib +from app.setup import delete_old_logs + from .main import app CURRENT_USER = "api-key|testing" +ENTRY_EXAMPLES = [ + { + "l_type": "info", + "t_type": "create", + "message": "User Max Mustermann created", + "environment": "dev", + }, + { + "l_type": "info", + "t_type": "update", + "message": "User Max Mustermann updated", + "environment": "dev", + }, + { + "l_type": "info", + "t_type": "create", + "author": "auth|max_muster", + "message": "User Max Mustermann created a Unit", + "object_reference": "1", + "environment": "dev", + }, + { + "l_type": "info", + "t_type": "update", + "author": "auth|max_muster", + "message": "User Max Mustermann updated Unit 1", + "object_reference": "1", + "previous_object": {"name": "Unit 1"}, + "environment": "prod", + }, + { + "l_type": "warning", + "t_type": "delete", + "message": "User Max Mustermann deleted", + "environment": "prod", + }, +] @contextlib.contextmanager -def app_context(self, name: str = "Testing"): - app_id = self.create_app(name) +def app_context(self, name: str = "Testing", retention_days: int | None = None): + app_id = self.create_app(name, retention_days) try: yield app_id finally: @@ -23,45 +65,8 @@ def app_context(self, name: str = "Testing"): @contextlib.contextmanager def log_examples(self): - LOG_EXAMPLES = [ - { - "l_type": "info", - "t_type": "create", - "message": "User Max Mustermann created", - "environment": "dev", - }, - { - "l_type": "info", - "t_type": "update", - "message": "User Max Mustermann updated", - "environment": "dev", - }, - { - "l_type": "info", - "t_type": "create", - "author": "auth|max_muster", - "message": "User Max Mustermann created a Unit", - "object_reference": "1", - "environment": "dev", - }, - { - "l_type": "info", - "t_type": "update", - "author": "auth|max_muster", - "message": "User Max Mustermann updated Unit 1", - "object_reference": "1", - "previous_object": {"name": "Unit 1"}, - "environment": "prod", - }, - { - "l_type": "warning", - "t_type": "delete", - "message": "User Max Mustermann deleted", - "environment": "prod", - }, - ] with app_context(self) as app_id: - for entry in LOG_EXAMPLES: + for entry in ENTRY_EXAMPLES: self.log_message({"application": app_id, **entry}) yield app_id @@ -86,6 +91,7 @@ class TestAPI: global CURRENT_USER return CURRENT_USER + self.db_instance = get_db_test() app.dependency_overrides[get_db] = get_db_test app.dependency_overrides[verify] = get_test_sub self.c = GenericClient(app) @@ -94,8 +100,8 @@ class TestAPI: drop_database(self.engine.url) # HELPERS - def create_app(self, name: str = "Testing"): - re = self.c.post("/app/", {"name": name}) + def create_app(self, name: str = "Testing", retention_days: int | None = None): + re = self.c.post("/app/", {"name": name, "retention_days": retention_days}) return re["id"] def destroy_app(self, app_id): @@ -260,3 +266,29 @@ class TestAPI: re = self.c.get("/log/?application=" + str(app_id)) assert re["total"] == 0 + + def test_retention_delete(self): + sess = next(self.db_instance) + + with app_context(self, retention_days=2) as app_id: + for i in range(5): + sess.add( + LogEntry( + application=app_id, + created_at=datetime.now() - timedelta(days=i), + created_by_id=CURRENT_USER, + ) + ) + sess.commit() + + assert sess.query(LogEntry).count() == 5 + + re = self.c.get("/log/?application=" + str(app_id)) + assert re["total"] == 5 + + delete_old_logs(sess) + + assert sess.query(LogEntry).count() == 2 + + # delete all logs + re = self.c.delete("/log/?application=" + str(app_id), r_code=200) diff --git a/requirements.txt b/requirements.txt index f87602c..99922f9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -34,3 +34,6 @@ pluggy==1.5.0 # pytest pytest==8.3.4 # pytest fastapi-filters==0.2.9 # Filters + +APScheduler==3.11.0 # Scheduler for deletion +tzlocal==5.2 # Scheduler for deletion