feat: implement attachment handling in email ingestion and add tests

This commit is contained in:
Ashwin 2026-09-27 15:13:19 +05:30
parent 5c1ffa802f
commit 6a43836566
5 changed files with 194 additions and 3 deletions

View file

@ -15,9 +15,14 @@ import logging
from dataclasses import dataclass
from typing import Optional
from django.core.exceptions import SuspiciousFileOperation
from django.core.files.base import ContentFile
from django.db import transaction
from django.utils.text import get_valid_filename
from cases.models import Case, EmailMessage, InboundMailbox
from common.models import Attachments
from common.utils import ATTACHMENT_MAX_BYTES, fit_attachment_file_name
from .contacts import resolve_contact
from .parser import ParsedEmail
@ -70,6 +75,36 @@ def _record_email_message(
return obj
def _store_attachments(parsed: ParsedEmail, case: Case) -> None:
"""Save the email's files on the case, under the same limits as an upload.
A file over the size cap is skipped rather than failing the email: the
ticket still opens, and refusing would only make the provider retry.
"""
label_length = Attachments._meta.get_field("file_name").max_length
for parsed_file in parsed.attachments:
if len(parsed_file.payload) > ATTACHMENT_MAX_BYTES:
logger.warning(
"Skipped inbound attachment of %d bytes on case %s",
len(parsed_file.payload),
case.id,
)
continue
label = fit_attachment_file_name(parsed_file.filename, label_length)
# The sender chose this name. Storage refuses one that cleans down to
# nothing, and that refusal must not fail the email.
try:
storage_name = get_valid_filename(label)
except SuspiciousFileOperation:
storage_name = "attachment"
Attachments.objects.create(
content_object=case,
org=case.org,
file_name=label,
attachment=ContentFile(parsed_file.payload, name=storage_name),
)
def _sender_is_on_case(parsed: ParsedEmail, case: Case) -> bool:
"""Whether the From address is already one of the case's contacts.
@ -155,6 +190,7 @@ def ingest(parsed: ParsedEmail, mailbox: InboundMailbox) -> IngestResult:
row = _record_email_message(
parsed=parsed, mailbox=mailbox, case=existing_case
)
_store_attachments(parsed, existing_case)
if (
contact is not None
and not existing_case.contacts.filter(pk=contact.pk).exists()
@ -208,6 +244,7 @@ def ingest(parsed: ParsedEmail, mailbox: InboundMailbox) -> IngestResult:
case.contacts.add(contact)
row = _record_email_message(parsed=parsed, mailbox=mailbox, case=case)
_store_attachments(parsed, case)
from cases.signals import emit_email_received_activity

View file

@ -0,0 +1,154 @@
"""Files attached to inbound support email land on the ticket.
The parser always extracted them; the pipeline dropped them on the floor, so a
customer's screenshot or invoice PDF was lost without a trace.
"""
from __future__ import annotations
from email.message import EmailMessage as StdlibEmailMessage
import pytest
from django.contrib.contenttypes.models import ContentType
from cases.inbound.parser import ParsedAttachment, parse_raw_email
from cases.inbound.pipeline import ingest
from cases.models import Case, InboundMailbox
from common.models import Attachments
from common.utils import ATTACHMENT_MAX_BYTES
from conftest import rls_org
def _mailbox(org):
with rls_org(org):
return InboundMailbox.objects.create(
org=org,
address="support@acme.com",
provider="ses",
webhook_secret="test-secret",
default_priority="Normal",
is_active=True,
)
def _raw(*, message_id="<m1@example.com>", in_reply_to="", files=(), headers=()):
msg = StdlibEmailMessage()
msg["From"] = "Customer <user@example.com>"
msg["To"] = "support@acme.com"
msg["Subject"] = "Broken export"
msg["Date"] = "Sat, 9 May 2026 12:00:00 +0000"
msg["Message-ID"] = message_id
if in_reply_to:
msg["In-Reply-To"] = in_reply_to
for name, value in headers:
msg[name] = value
msg.set_content("See attached.")
for filename, payload in files:
msg.add_attachment(
payload, maintype="application", subtype="pdf", filename=filename
)
return msg.as_bytes()
def _case_attachments(case):
return Attachments.objects.filter(
content_type=ContentType.objects.get_for_model(Case), object_id=case.id
)
@pytest.mark.django_db
class TestInboundAttachments:
def test_new_ticket_keeps_the_attachment(self, org_a):
result = ingest(
parse_raw_email(_raw(files=[("invoice.pdf", b"%PDF-1.4 data")])),
_mailbox(org_a),
)
[att] = _case_attachments(result.case)
assert att.file_name == "invoice.pdf"
assert att.org_id == org_a.id
with att.attachment.open("rb") as fh:
assert fh.read() == b"%PDF-1.4 data"
def test_threaded_reply_attaches_to_the_existing_ticket(self, org_a):
mailbox = _mailbox(org_a)
first = ingest(parse_raw_email(_raw()), mailbox)
reply = ingest(
parse_raw_email(
_raw(
message_id="<m2@example.com>",
in_reply_to="<m1@example.com>",
files=[("log.pdf", b"trace")],
)
),
mailbox,
)
assert reply.case == first.case
assert [a.file_name for a in _case_attachments(first.case)] == ["log.pdf"]
def test_dropped_email_stores_nothing(self, org_a):
result = ingest(
parse_raw_email(
_raw(
files=[("x.pdf", b"x")],
headers=[("Auto-Submitted", "auto-replied")],
)
),
_mailbox(org_a),
)
assert result.dropped is True
assert not Attachments.objects.exists()
def test_provider_retry_does_not_duplicate_attachments(self, org_a):
mailbox = _mailbox(org_a)
raw = _raw(files=[("invoice.pdf", b"data")])
first = ingest(parse_raw_email(raw), mailbox)
ingest(parse_raw_email(raw), mailbox)
assert _case_attachments(first.case).count() == 1
def test_oversized_file_is_skipped_and_the_ticket_still_opens(self, org_a):
parsed = parse_raw_email(_raw(files=[("small.pdf", b"ok")]))
parsed.attachments.append(
ParsedAttachment(
filename="huge.bin",
content_type="application/octet-stream",
payload=b"\0" * (ATTACHMENT_MAX_BYTES + 1),
)
)
result = ingest(parsed, _mailbox(org_a))
assert result.created_case is True
assert [a.file_name for a in _case_attachments(result.case)] == ["small.pdf"]
@pytest.mark.parametrize(
"hostile, label",
[
("../../etc/passwd", "passwd"),
("..", "attachment"),
("", "attachment"),
# Nothing survives Django's storage-name cleaning, which then raises
# SuspiciousFileOperation and would fail the whole email.
("()", "()"),
],
)
def test_sender_controlled_filename_cannot_escape_storage(
self, org_a, hostile, label
):
parsed = parse_raw_email(_raw())
parsed.attachments.append(
ParsedAttachment(
filename=hostile, content_type="text/plain", payload=b"data"
)
)
result = ingest(parsed, _mailbox(org_a))
[att] = _case_attachments(result.case)
assert att.file_name == label
assert att.attachment.name.startswith("attachments/")
assert ".." not in att.attachment.name

View file

@ -531,7 +531,7 @@ def fit_attachment_file_name(raw_name, max_length):
``../../../etc/passwd`` has no business being rendered as one.
"""
name = (raw_name or "").replace("\\", "/").rsplit("/", 1)[-1].strip()
if not name:
if name in ("", ".", ".."):
return "attachment"
if len(name) <= max_length:
return name

View file

@ -4,7 +4,7 @@ build-backend = "hatchling.build"
[project]
name = "django-crm"
version = "1.14.0"
version = "1.14.1"
# The one-line summary PyPI shows under the project title and in search results,
# so it carries the terms people actually search for ("open source CRM",
# "self-hosted CRM", "Django CRM") and names the modules rather than the

View file

@ -681,7 +681,7 @@ wheels = [
[[package]]
name = "django-crm"
version = "1.14.0"
version = "1.14.1"
source = { editable = "." }
dependencies = [
{ name = "cairocffi" },