diff --git a/backend/Makefile b/backend/Makefile index 08ab1ae0..5ccd991c 100644 --- a/backend/Makefile +++ b/backend/Makefile @@ -66,7 +66,7 @@ stop-worker: -$(PYTHON) -m celery -A core control shutdown \ --destination=worker@$(shell hostname) || true -test: test-models test-header-validation-task test-syntax-task test-syntax-header-validation-task test-schema-task test-magic-and-av-task test-utils +test: test-models test-header-validation-task test-syntax-task test-syntax-header-validation-task test-schema-task test-magic-and-av-task test-utils test-file-retention-task test-models: MEDIA_ROOT=./apps/ifc_validation/fixtures $(PYTHON) manage.py test apps/ifc_validation_models --settings apps.ifc_validation_models.test_settings --debug-mode --verbosity 3 @@ -84,7 +84,10 @@ test-schema-task: MEDIA_ROOT=./apps/ifc_validation/fixtures $(PYTHON) manage.py test apps.ifc_validation.tests.tests_schema_validation_task --settings apps.ifc_validation.test_settings --debug-mode --verbosity 3 test-magic-and-av-task: - MEDIA_ROOT=./apps/ifc_validation/fixtures $(PYTHON) manage.py test apps.ifc_validation.tests.tests_magic_clamav_task --settings apps.ifc_validation.test_settings --debug-mode --verbosity 3 + MEDIA_ROOT=./apps/ifc_validation/fixtures $(PYTHON) manage.py test apps.ifc_validation.tests.tests_magic_clamav_task --settings apps.ifc_validation.test_settings --debug-mode --verbosity 3 + +test-file-retention-task: + MEDIA_ROOT=./apps/ifc_validation/fixtures $(PYTHON) manage.py test apps.ifc_validation.tests.tests_file_retention_task --settings apps.ifc_validation.test_settings --debug-mode --verbosity 3 test-utils: $(PYTHON) manage.py test core.tests.test_utils --debug-mode --verbosity 3 diff --git a/backend/apps/ifc_validation/management/commands/apply_file_retention.py b/backend/apps/ifc_validation/management/commands/apply_file_retention.py index 85b1aa96..25f0505c 100644 --- a/backend/apps/ifc_validation/management/commands/apply_file_retention.py +++ b/backend/apps/ifc_validation/management/commands/apply_file_retention.py @@ -92,13 +92,15 @@ def handle(self, *args, **options): file_path = get_absolute_file_path(request.file.name) original_size = os.path.getsize(file_path) except FileNotFoundError: - logger.warning(f"File not found for ValidationRequest id={request.id} ({request.file.name}) - skipping") - skipped += 1 - continue + file_path = None + original_size = 0 + logger.warning(f"File not found for ValidationRequest id={request.id} ({request.file.name})") + # skipped += 1 + # continue # original and target names original_name = request.file.name - gz_filename = file_path + '.gz' + gz_filename = file_path + '.gz' if file_path else original_name + '.gz' gz_name_only = original_name + '.gz' # only report what would happen @@ -118,6 +120,11 @@ def handle(self, *args, **options): # execute action if action == 'archive': + if not file_path: + logger.warning(f"File not found for ValidationRequest id={request.id} ({request.file.name}) - skipping") + skipped += 1 + continue + try: # create gzip archive with open(file_path, 'rb') as f_in, gzip.open(gz_filename, 'wb') as f_out: @@ -154,8 +161,10 @@ def handle(self, *args, **options): with transaction.atomic(): original_name = request.file.name request.file = None - request.save(update_fields=['file']) - os.remove(file_path) + request.file_removed = timezone.now() + request.save(update_fields=['file', 'file_removed']) + if file_path: + os.remove(file_path) logger.info(f"Removed file and updated Validation Request with id={request.id}: {original_name}") total_savings += original_size diff --git a/backend/apps/ifc_validation/test_settings.py b/backend/apps/ifc_validation/test_settings.py index 20fb7053..bf666971 100644 --- a/backend/apps/ifc_validation/test_settings.py +++ b/backend/apps/ifc_validation/test_settings.py @@ -6,6 +6,7 @@ INSTALLED_APPS = [ "django.contrib.auth", "django.contrib.contenttypes", + "apps.ifc_validation", "apps.ifc_validation_models" ] diff --git a/backend/apps/ifc_validation/tests/tests_file_retention_task.py b/backend/apps/ifc_validation/tests/tests_file_retention_task.py new file mode 100644 index 00000000..aa0e0d76 --- /dev/null +++ b/backend/apps/ifc_validation/tests/tests_file_retention_task.py @@ -0,0 +1,136 @@ +from io import StringIO + +from django.test import TransactionTestCase +from django.contrib.auth.models import User + +from apps.ifc_validation_models.models import * + +from ..tasks.file_retention_tasks import apply_file_retention + + +class ApplyFileRetentionTaskTestCase(TransactionTestCase): + + def set_user_context(): + user, _ = User.objects.get_or_create(id=1, defaults={'username': 'SYSTEM', 'is_active': True}) + set_user_context(user) + + def test_apply_file_retention_archive_updates_file_name(self): + + # arrange + ApplyFileRetentionTaskTestCase.set_user_context() + request = ValidationRequest.objects.create( + file_name='valid_file.ifc', + file='valid_file.ifc', + size=1 + ) + request.created = timezone.now() - timezone.timedelta(days=250) + request.save() + + # act + task = apply_file_retention(dry_run=False, action="archive") + + # assert + request = ValidationRequest.objects.get(id=request.id) + self.assertIsNotNone(request.file) + self.assertEquals('valid_file.ifc.gz', request.file.name) + + def test_apply_file_retention_archive_in_dry_mode_leaves_record_intact(self): + + # arrange + ApplyFileRetentionTaskTestCase.set_user_context() + request = ValidationRequest.objects.create( + file_name='valid_file.ifc', + file='valid_file.ifc', + size=1 + ) + request.created = timezone.now() - timezone.timedelta(days=250) + request.save() + + # act + task = apply_file_retention(dry_run=True, action="archive") + + # assert + request = ValidationRequest.objects.get(id=request.id) + self.assertIsNone(request.file_removed) + self.assertEquals('valid_file.ifc', request.file) + + def test_apply_file_retention_remove_sets_file_removed_field(self): + + # arrange + ApplyFileRetentionTaskTestCase.set_user_context() + request = ValidationRequest.objects.create( + file_name='valid_file.ifc', + file='valid_file.ifc', + size=1 + ) + request.created = timezone.now() - timezone.timedelta(days=250) + request.save() + + # act + task = apply_file_retention(dry_run=False, action="remove") + + # assert + request = ValidationRequest.objects.get(id=request.id) + self.assertIsNotNone(request.file_removed) + + def test_apply_file_retention_remove_retains_removed_field(self): + + # arrange + ApplyFileRetentionTaskTestCase.set_user_context() + request = ValidationRequest.objects.create( + file_name='valid_file.ifc', + file='valid_file.ifc', + size=1 + ) + orig_file_removed = timezone.now() - timezone.timedelta(days=50) + request.created = timezone.now() - timezone.timedelta(days=250) + request.file_removed = orig_file_removed + request.file = None + request.save() + + # act + task = apply_file_retention(dry_run=False, action="remove") + + # assert + request = ValidationRequest.objects.get(id=request.id) + self.assertIsNotNone(request.file_removed) + self.assertEquals(orig_file_removed, request.file_removed) + + def test_apply_file_retention_remove_empties_file_field(self): + + # arrange + ApplyFileRetentionTaskTestCase.set_user_context() + request = ValidationRequest.objects.create( + file_name='valid_file.ifc', + file='valid_file.ifc', + size=1 + ) + request.created = timezone.now() - timezone.timedelta(days=250) + request.save() + + # act + task = apply_file_retention(dry_run=False, action="remove") + + # assert + request = ValidationRequest.objects.get(id=request.id) + self.assertEquals('', request.file) + + def test_apply_file_retention_remove_in_dry_mode_leaves_record_intact(self): + + # arrange + ApplyFileRetentionTaskTestCase.set_user_context() + request = ValidationRequest.objects.create( + file_name='valid_file.ifc', + file='valid_file.ifc', + size=1 + ) + request.created = timezone.now() - timezone.timedelta(days=250) + request.save() + + # act + task = apply_file_retention(dry_run=True, action="remove") + + # assert + request = ValidationRequest.objects.get(id=request.id) + self.assertIsNone(request.file_removed) + self.assertNotEquals('', request.file)