IVS-815 - BUG fix: file_removed field not populated in file_retention job

backend/Makefile

@@ -66,7 +66,7 @@ stop-worker:
 	-$(PYTHON) -m celery -A core control shutdown \
 	    --destination=worker@$(shell hostname) || true
 
-test: test-models test-header-validation-task test-syntax-task test-syntax-header-validation-task test-schema-task test-magic-and-av-task test-utils
+test: test-models test-header-validation-task test-syntax-task test-syntax-header-validation-task test-schema-task test-magic-and-av-task test-utils test-file-retention-task
 
 test-models:
 	MEDIA_ROOT=./apps/ifc_validation/fixtures $(PYTHON) manage.py test apps/ifc_validation_models --settings apps.ifc_validation_models.test_settings --debug-mode --verbosity 3
@@ -84,7 +84,10 @@ test-schema-task:
 	MEDIA_ROOT=./apps/ifc_validation/fixtures $(PYTHON) manage.py test apps.ifc_validation.tests.tests_schema_validation_task --settings apps.ifc_validation.test_settings --debug-mode --verbosity 3
 
 test-magic-and-av-task:
-	MEDIA_ROOT=./apps/ifc_validation/fixtures $(PYTHON) manage.py test apps.ifc_validation.tests.tests_magic_clamav_task --settings apps.ifc_validation.test_settings --debug-mode --verbosity 3
+	MEDIA_ROOT=./apps/ifc_validation/fixtures $(PYTHON) manage.py test apps.ifc_validation.tests.tests_magic_clamav_task --settings apps.ifc_validation.test_settings --debug-mode --verbosity 3	
+
+test-file-retention-task:
+	MEDIA_ROOT=./apps/ifc_validation/fixtures $(PYTHON) manage.py test apps.ifc_validation.tests.tests_file_retention_task --settings apps.ifc_validation.test_settings --debug-mode --verbosity 3
 
 test-utils:
 	$(PYTHON) manage.py test core.tests.test_utils --debug-mode --verbosity 3

backend/apps/ifc_validation/management/commands/apply_file_retention.py

@@ -92,13 +92,15 @@ def handle(self, *args, **options):
                 file_path = get_absolute_file_path(request.file.name)
                 original_size = os.path.getsize(file_path)
             except FileNotFoundError:
-                logger.warning(f"File not found for ValidationRequest id={request.id} ({request.file.name}) - skipping")
-                skipped += 1
-                continue
+                file_path = None
+                original_size = 0
+                logger.warning(f"File not found for ValidationRequest id={request.id} ({request.file.name})")
+                # skipped += 1
+                # continue
 
             # original and target names
             original_name = request.file.name
-            gz_filename = file_path + '.gz'
+            gz_filename = file_path + '.gz' if file_path else original_name + '.gz'
             gz_name_only = original_name + '.gz'
 
             # only report what would happen
@@ -118,6 +120,11 @@ def handle(self, *args, **options):
             # execute action
             if action == 'archive':
 
+                if not file_path:
+                    logger.warning(f"File not found for ValidationRequest id={request.id} ({request.file.name}) - skipping")
+                    skipped += 1
+                    continue
+
                 try:
                     # create gzip archive
                     with open(file_path, 'rb') as f_in, gzip.open(gz_filename, 'wb') as f_out:
@@ -154,8 +161,10 @@ def handle(self, *args, **options):
                     with transaction.atomic():
                         original_name = request.file.name
                         request.file = None
-                        request.save(update_fields=['file'])
-                        os.remove(file_path)
+                        request.file_removed = timezone.now()
+                        request.save(update_fields=['file', 'file_removed'])
+                        if file_path:
+                            os.remove(file_path)
 
                     logger.info(f"Removed file and updated Validation Request with id={request.id}: {original_name}")
                     total_savings += original_size

backend/apps/ifc_validation/test_settings.py

@@ -6,6 +6,7 @@
 INSTALLED_APPS = [
     "django.contrib.auth",
     "django.contrib.contenttypes",
+    "apps.ifc_validation",
     "apps.ifc_validation_models"
 ]
 

backend/apps/ifc_validation/tests/tests_file_retention_task.py

@@ -0,0 +1,136 @@
+from io import StringIO
+
+from django.test import TransactionTestCase
+from django.contrib.auth.models import User
+
+from apps.ifc_validation_models.models import *
+
+from ..tasks.file_retention_tasks import apply_file_retention
+
+
+class ApplyFileRetentionTaskTestCase(TransactionTestCase):
+
+    def set_user_context():
+        user, _ = User.objects.get_or_create(id=1, defaults={'username': 'SYSTEM', 'is_active': True})
+        set_user_context(user)
+
+    def test_apply_file_retention_archive_updates_file_name(self):
+
+        # arrange
+        ApplyFileRetentionTaskTestCase.set_user_context()
+        request = ValidationRequest.objects.create(
+            file_name='valid_file.ifc',
+            file='valid_file.ifc', 
+            size=1
+        )
+        request.created = timezone.now() - timezone.timedelta(days=250)
+        request.save()
+
+        # act
+        task = apply_file_retention(dry_run=False, action="archive")
+
+        # assert
+        request = ValidationRequest.objects.get(id=request.id)
+        self.assertIsNotNone(request.file)
+        self.assertEquals('valid_file.ifc.gz', request.file.name)
+
+    def test_apply_file_retention_archive_in_dry_mode_leaves_record_intact(self):
+
+        # arrange
+        ApplyFileRetentionTaskTestCase.set_user_context()
+        request = ValidationRequest.objects.create(
+            file_name='valid_file.ifc',
+            file='valid_file.ifc', 
+            size=1
+        )
+        request.created = timezone.now() - timezone.timedelta(days=250)
+        request.save()
+
+        # act
+        task = apply_file_retention(dry_run=True, action="archive")
+
+        # assert
+        request = ValidationRequest.objects.get(id=request.id)
+        self.assertIsNone(request.file_removed)
+        self.assertEquals('valid_file.ifc', request.file)
+
+    def test_apply_file_retention_remove_sets_file_removed_field(self):
+
+        # arrange
+        ApplyFileRetentionTaskTestCase.set_user_context()
+        request = ValidationRequest.objects.create(
+            file_name='valid_file.ifc',
+            file='valid_file.ifc', 
+            size=1
+        )
+        request.created = timezone.now() - timezone.timedelta(days=250)
+        request.save()
+
+        # act
+        task = apply_file_retention(dry_run=False, action="remove")
+
+        # assert
+        request = ValidationRequest.objects.get(id=request.id)
+        self.assertIsNotNone(request.file_removed)
+
+    def test_apply_file_retention_remove_retains_removed_field(self):
+
+        # arrange
+        ApplyFileRetentionTaskTestCase.set_user_context()
+        request = ValidationRequest.objects.create(
+            file_name='valid_file.ifc',
+            file='valid_file.ifc', 
+            size=1
+        )
+        orig_file_removed = timezone.now() - timezone.timedelta(days=50)
+        request.created = timezone.now() - timezone.timedelta(days=250)
+        request.file_removed = orig_file_removed
+        request.file = None
+        request.save()
+
+        # act
+        task = apply_file_retention(dry_run=False, action="remove")
+
+        # assert
+        request = ValidationRequest.objects.get(id=request.id)
+        self.assertIsNotNone(request.file_removed)
+        self.assertEquals(orig_file_removed, request.file_removed)
+
+    def test_apply_file_retention_remove_empties_file_field(self):
+
+        # arrange
+        ApplyFileRetentionTaskTestCase.set_user_context()
+        request = ValidationRequest.objects.create(
+            file_name='valid_file.ifc',
+            file='valid_file.ifc', 
+            size=1
+        )
+        request.created = timezone.now() - timezone.timedelta(days=250)
+        request.save()
+
+        # act
+        task = apply_file_retention(dry_run=False, action="remove")
+
+        # assert
+        request = ValidationRequest.objects.get(id=request.id)
+        self.assertEquals('', request.file)
+
+    def test_apply_file_retention_remove_in_dry_mode_leaves_record_intact(self):
+
+        # arrange
+        ApplyFileRetentionTaskTestCase.set_user_context()
+        request = ValidationRequest.objects.create(
+            file_name='valid_file.ifc',
+            file='valid_file.ifc', 
+            size=1
+        )
+        request.created = timezone.now() - timezone.timedelta(days=250)
+        request.save()
+
+        # act
+        task = apply_file_retention(dry_run=True, action="remove")
+
+        # assert
+        request = ValidationRequest.objects.get(id=request.id)
+        self.assertIsNone(request.file_removed)
+        self.assertNotEquals('', request.file)