IVS-605 Run ClamAV and file magic detection to erase potentially harmful content

backend/Makefile

@@ -66,7 +66,7 @@ stop-worker:
 	-$(PYTHON) -m celery -A core control shutdown \
 	    --destination=worker@$(shell hostname) || true
 
-test: test-models test-header-validation-task test-syntax-task test-syntax-header-validation-task test-schema-task
+test: test-models test-header-validation-task test-syntax-task test-syntax-header-validation-task test-schema-task test-magic-and-av-task
 
 test-models:
 	MEDIA_ROOT=./apps/ifc_validation/fixtures $(PYTHON) manage.py test apps/ifc_validation_models --settings apps.ifc_validation_models.test_settings --debug-mode --verbosity 3
@@ -83,6 +83,9 @@ test-syntax-task:
 test-schema-task:
 	MEDIA_ROOT=./apps/ifc_validation/fixtures $(PYTHON) manage.py test apps.ifc_validation.tests.tests_schema_validation_task --settings apps.ifc_validation.test_settings --debug-mode --verbosity 3
 
+test-magic-and-av-task:
+	MEDIA_ROOT=./apps/ifc_validation/fixtures $(PYTHON) manage.py test apps.ifc_validation.tests.tests_magic_clamav_task --settings apps.ifc_validation.test_settings --debug-mode --verbosity 3
+
 archive-dry-run:
 	$(PYTHON) manage.py archive_requests --days 180 --all --dry-run
 

backend/apps/ifc_validation/admin.py

@@ -450,7 +450,8 @@ class ModelAdmin(BaseAdmin, NonAdminAddable):
             "status_industry_practices",
             "status_mvd",
             "status_bsdd",
-            "status_signatures"
+            "status_signatures",
+            "status_magic_clamav",
         ]}),
         ('Auditing Information', {"classes": ("wide"), "fields": [("created",), ("updated")]})
     ]

backend/apps/ifc_validation/chart_views.py

@@ -65,6 +65,7 @@
     "prereq": "#b4acb4",
     "digital_signatures": "#73d0d8",
     "inst_completion": "#e76565",
+    "magic_and_av": "#a29bfe",
 }
 
 SYNTAX_TASK_TYPES = {
@@ -83,6 +84,7 @@
     "INDUSTRY": ("Industry", COLORS["industry"]),
     "PREREQ": ("Prereq", COLORS["prereq"]),
     "INST_COMPLETION": ("Inst Completion", COLORS["inst_completion"]),
+    "MAGIC_AND_CLAMAV": ("Magic/AV", COLORS["magic_and_av"]),
 }
 
 SECONDS_PER_MINUTE = 60

backend/apps/ifc_validation/tasks/__init__.py

@@ -13,7 +13,8 @@
     normative_rules_ip_validation_subtask, 
     bsdd_validation_subtask,
     industry_practices_subtask, 
-    instance_completion_subtask
+    instance_completion_subtask,
+    magic_clamav_subtask
 )
 
 __all__ = [
@@ -28,4 +29,5 @@
     "normative_rules_ip_validation_subtask",
     "industry_practices_subtask",
     "instance_completion_subtask",
+    "magic_clamav_subtask"
 ]

backend/apps/ifc_validation/tasks/check_programs.py

@@ -1,10 +1,15 @@
 import os
 import sys
 import json
+import shutil
 import subprocess
 from typing import List
 from dataclasses import dataclass
 
+# pip install filetype
+import filetype
+from filetype.types import archive
+
 from apps.ifc_validation_models.settings import TASK_TIMEOUT_LIMIT
 from apps.ifc_validation_models.models import ValidationTask
 
@@ -112,6 +117,57 @@ def check_header(context:TaskContext):
     return context
 
 
+def check_magic_and_clamav(context:TaskContext):
+    result = {}
+    ty = filetype.guess(context.file_path)
+    if type(ty) in (type(None), archive.Zip):
+        # happy path, continue
+        # some support for zipbombs
+        clamscan = shutil.which('clamscan')
+        if clamscan:
+            proc = run_subprocess(
+                task=context.task,
+                command=[
+                    clamscan, 
+                    '--alert-exceeds-max', '--max-recursion=2', '--max-files=10', '--max-scansize=256M', '--max-filesize=128M', 
+                    '--stdout', 
+                    context.file_path] 
+            )
+            if proc.returncode != 0:
+                result = {
+                    'invalid': f'suspicious file\n\n{proc.stdout}\n{proc.stderr}'
+                }
+            else:
+                result = {
+                    'valid': 'unknown type' if ty is None else ty.mime
+                }
+        else:
+            print('WARNING: clamscan not installed')
+            result = {
+                    'warn': 'clamscan not installed'
+                }
+    else:
+        try:
+            mime = ty.mime
+        except:
+            mime = 'unknown mime type'
+        result = {
+            'invalid': mime
+        }
+    if 'invalid' in result:
+        print('REMOVING:', context.file_path)
+        # we do not unlink() the file because it would create DB issues
+        # when a file with the exact same name is reuploaded and it is not
+        # made unique.
+        with open(context.file_path, 'w'):
+            pass
+    context.result = {
+        "success": 'warn' not in result,
+        "valid": 'invalid' not in result,
+        'output': next(iter(result.values())),
+    }
+    return context
+
 def check_digital_signatures(context:TaskContext):
     proc = run_subprocess(
         task=context.task,

backend/apps/ifc_validation/tasks/configs.py

@@ -44,17 +44,18 @@ def _load_function(module, prefix, type):
     )
 
 # define task info
-header_syntax       = make_task(type=ValidationTask.Type.HEADER_SYNTAX, increment=5, field='status_header_syntax', stage="serial")
-header              = make_task(type=ValidationTask.Type.HEADER,        increment=10, field='status_header',      stage="serial")
-syntax              = make_task(type=ValidationTask.Type.SYNTAX,        increment=5,  field='status_syntax',      stage="serial")
-prerequisites       = make_task(type=ValidationTask.Type.PREREQUISITES, increment=10, field='status_prereq',      stage="serial")
-schema              = make_task(type=ValidationTask.Type.SCHEMA,        increment=10, field='status_schema')
-digital_signatures  = make_task(type=ValidationTask.Type.DIGITAL_SIGNATURES, increment=5,  field='status_signatures')
-bsdd                = make_task(type=ValidationTask.Type.BSDD,          increment=0,  field='status_bsdd')
-normative_ia        = make_task(type=ValidationTask.Type.NORMATIVE_IA,  increment=20, field='status_ia')
-normative_ip        = make_task(type=ValidationTask.Type.NORMATIVE_IP,  increment=20, field='status_ip')
-industry_practices  = make_task(type=ValidationTask.Type.INDUSTRY_PRACTICES, increment=10, field='status_industry_practices')
-instance_completion = make_task(type=ValidationTask.Type.INSTANCE_COMPLETION, increment=5, field=None,           stage="final")
+magic_clamav        = make_task(type=ValidationTask.Type.MAGIC_AND_CLAMAV,    increment=5,  field='status_magic_clamav',  stage="serial")
+header_syntax       = make_task(type=ValidationTask.Type.HEADER_SYNTAX,       increment=5,  field='status_header_syntax', stage="serial")
+header              = make_task(type=ValidationTask.Type.HEADER,              increment=10, field='status_header',        stage="serial")
+syntax              = make_task(type=ValidationTask.Type.SYNTAX,              increment=5,  field='status_syntax',        stage="serial")
+prerequisites       = make_task(type=ValidationTask.Type.PREREQUISITES,       increment=5,  field='status_prereq',        stage="serial")
+schema              = make_task(type=ValidationTask.Type.SCHEMA,              increment=10, field='status_schema')
+digital_signatures  = make_task(type=ValidationTask.Type.DIGITAL_SIGNATURES,  increment=5,  field='status_signatures')
+bsdd                = make_task(type=ValidationTask.Type.BSDD,                increment=0,  field='status_bsdd')
+normative_ia        = make_task(type=ValidationTask.Type.NORMATIVE_IA,        increment=20, field='status_ia')
+normative_ip        = make_task(type=ValidationTask.Type.NORMATIVE_IP,        increment=20, field='status_ip')
+industry_practices  = make_task(type=ValidationTask.Type.INDUSTRY_PRACTICES,  increment=10, field='status_industry_practices')
+instance_completion = make_task(type=ValidationTask.Type.INSTANCE_COMPLETION, increment=5,  field=None,                   stage="final")
 
 # block tasks on error
 post_tasks = [digital_signatures, schema, normative_ia, normative_ip, industry_practices, instance_completion]
@@ -64,10 +65,15 @@ def _load_function(module, prefix, type):
 
 # register
 ALL_TASKS = [
+    magic_clamav,
     header_syntax, header, syntax, prerequisites,
     schema, digital_signatures, bsdd,
     normative_ia, normative_ip, industry_practices, instance_completion,
 ]
+
+# av check blocks everything
+magic_clamav.blocks = [t for t in ALL_TASKS if t is not magic_clamav]
+
 class TaskRegistry:
     def __init__(self, config_map: dict[str, TaskConfig]):
         self._configs = config_map

backend/apps/ifc_validation/tasks/processing/__init__.py

@@ -10,4 +10,5 @@
 from .schema import process_schema
 from .header import process_header
 from .digital_signatures import process_digital_signatures
-from .bsdd import process_bsdd
+from .bsdd import process_bsdd
+from .magicav import process_magic_and_clamav

backend/apps/ifc_validation/tasks/processing/magicav.py

@@ -0,0 +1,13 @@
+from .. import TaskContext, with_model
+from apps.ifc_validation_models.models import Model
+
+
+def process_magic_and_clamav(context:TaskContext):
+    output, success, valid = (context.result.get(k) for k in ("output", "success", "valid"))
+
+    with with_model(context.request.id) as model:
+        agg_status = Model.Status.VALID if valid else Model.Status.INVALID
+        setattr(model, context.config.status_field.name, agg_status)
+
+        model.save(update_fields=[context.config.status_field.name])
+        return f'Magic and av check completed:\nsuccess = {success}\nvalid = {valid}\noutput = {output}'

backend/apps/ifc_validation/tasks/task_runner.py

@@ -203,6 +203,7 @@ def ifc_file_validation_task(self, id, file_name, *args, **kwargs):
     workflow_completed = on_workflow_completed.s(id=id, file_name=file_name)
 
     serial_tasks = chain(
+        magic_clamav_subtask.s(id=id, file_name=file_name),
         header_syntax_validation_subtask.s(id=id, file_name=file_name),
         header_validation_subtask.s(id=id, file_name=file_name),
         syntax_validation_subtask.s(id=id, file_name=file_name),
@@ -254,3 +255,5 @@ def ifc_file_validation_task(self, id, file_name, *args, **kwargs):
 bsdd_validation_subtask = task_factory(ValidationTask.Type.BSDD)
 
 industry_practices_subtask = task_factory(ValidationTask.Type.INDUSTRY_PRACTICES)
+
+magic_clamav_subtask = task_factory(ValidationTask.Type.MAGIC_AND_CLAMAV)

backend/apps/ifc_validation/tests/tests_magic_clamav_task.py

@@ -0,0 +1,70 @@
+import datetime
+
+from django.test import TransactionTestCase
+from django.contrib.auth.models import User
+
+from apps.ifc_validation_models.models import *
+from apps.ifc_validation.tasks.utils import get_absolute_file_path
+
+from ..tasks import magic_clamav_subtask
+
+class MagicClamAVTaskTestCase(TransactionTestCase):
+
+    def set_user_context():
+        user = User.objects.create(id=1, username='SYSTEM', is_active=True)
+        set_user_context(user)
+
+    def test_magic_clamav_task_detects_valid_ifc_file(self):
+
+        # arrange
+        MagicClamAVTaskTestCase.set_user_context()
+        request = ValidationRequest.objects.create(
+            file_name='valid_file.ifc',
+            file='valid_file.ifc', 
+            size=1
+        )
+        request.mark_as_initiated()
+
+        # act
+        task = magic_clamav_subtask(
+            prev_result={'is_valid': True, 'reason': 'test'}, 
+            id=request.id, 
+            file_name=request.file_name
+        )
+        print(task)
+
+        # assert
+        model = Model.objects.get(id=request.id)
+        self.assertIsNotNone(model)
+        self.assertEqual(model.status_magic_clamav, Model.Status.VALID)
+
+    def test_magic_clamav_task_detects_eicar_testfile(self):
+
+        # arrange
+        MagicClamAVTaskTestCase.set_user_context()
+        request = ValidationRequest.objects.create(
+            file_name='eicar_testfile.ifc',
+            file='eicar_testfile.ifc', 
+            size=68
+        )
+        request.mark_as_initiated()
+
+        # make sure the test file contains eicar test string
+        file_path = get_absolute_file_path(request.file.name)
+        with open(file_path, 'w') as f:
+            EICAR_TEST_STRING = 'X5O!P%@AP[4\PZX54(P^)7CC)7}$EICAR-STANDARD-ANTIVIRUS-TEST-FILE!$H+H*'
+            f.write(EICAR_TEST_STRING)
+
+        # act
+        task = magic_clamav_subtask(
+            prev_result={'is_valid': True, 'reason': 'test'}, 
+            id=request.id, 
+            file_name=request.file_name
+        )
+        print(task)
+
+        # assert
+        model = Model.objects.get(id=request.id)
+        self.assertIsNotNone(model)
+        self.assertEqual(model.status_magic_clamav, Model.Status.INVALID)
+

backend/apps/ifc_validation_bff/views_legacy.py

@@ -197,6 +197,7 @@ def format_request(request):
         ),
         "status_ind": "p" if (request.model is None or request.model.status_industry_practices is None) else request.model.status_industry_practices,
         "status_signatures": "p" if (request.model is None or request.model.status_signatures is None) else request.model.status_signatures,
+        "status_magic_clamav": "p" if (request.model is None or request.model.status_magic_clamav is None) else request.model.status_magic_clamav,
         "deleted": 0, # TODO
         "commit_id": None #  TODO
     }

backend/requirements.txt

@@ -46,6 +46,7 @@ shapely==2.1.1
 python-ranges==1.2.2
 pyproj==3.7.1
 python-dateutil==2.9.0.post0
+filetype==1.2.0
 
 # dev
 django-debug-toolbar==6.0.0

docker/backend/Dockerfile

@@ -72,8 +72,11 @@ RUN set -ex && \
         postgresql-client \
         netcat-openbsd \
         procps \
-        htop && \
+        htop \
+        clamav && \
     update-ca-certificates -f && \
+    # clamav initial setup
+    freshclam && \
     # cleanup
     apt-get -y clean && \
     apt-get autoremove -y && \

frontend/src/DashboardTable.js

@@ -434,9 +434,12 @@ export default function DashboardTable({ models }) {
                   }
 
                   <TableCell align="left">
-                    <Link href={`${FETCH_PATH}/api/download/${row.id}`} underline="hover" onClick={evt => evt.stopPropagation()}>
-                      {'Download file'}
-                    </Link>
+                  { 
+                    (row.status_magic_clamav != 'i') ?
+                      <Link href={`${FETCH_PATH}/api/download/${row.id}`} underline="hover" onClick={evt => evt.stopPropagation()}>
+                        {'Download file'}
+                      </Link> : 'File unavailable'
+                  }
                   </TableCell>
                 </TableRow>
               );