From 1cc49ffb8b1e391c7901766c17dcec4689f563ed Mon Sep 17 00:00:00 2001 From: Md Nurnobi Date: Wed, 19 Aug 2026 13:36:05 -0700 Subject: [PATCH] PHASE-006: promote reader fidelity candidate to 1.3.4-rc.2 --- .github/workflows/ci.yml | 4 +- CHANGELOG.md | 10 +- VERSION | 2 +- design/DESIGN_MANIFEST.json | 2 +- docs/DEPENDENCY_REVIEW.md | 12 +- docs/FORENSIC_FILE_INVENTORY.json | 231 +++++++---- docs/PUBLISHING.md | 24 +- docs/RELEASE_NOTES_1.3.4.md | 62 +-- docs/RELEASE_PROCESS.md | 2 +- .../03_UPDATE_PHASE_COMPLETION_LOG.md | 33 +- documents/ADMIN_GUIDE.md | 2 +- documents/BASELINE.md | 2 +- documents/DOCUMENTATION_MANIFEST.json | 78 ++-- documents/DOCUMENTATION_POLICY.md | 2 +- documents/HOW_TO_USE.md | 2 +- documents/README.md | 38 +- documents/USER_MANUAL.md | 2 +- .../design/ACCESSIBILITY_SPECIFICATION.md | 2 +- documents/design/COMPONENT_MATRIX.md | 2 +- documents/design/FUTURE_UI_ROADMAP.md | 2 +- documents/design/IMPLEMENTATION_STATUS.md | 2 +- documents/design/RESPONSIVE_SPECIFICATION.md | 2 +- documents/design/SCREEN_CATALOG.md | 2 +- documents/design/UI_FOUNDATION.md | 2 +- documents/phases/PHASE-000-BASELINE.md | 2 +- .../PHASE-001-UI-DESIGN-INTAKE-BASELINE.md | 2 +- ...RED-UI-FOUNDATION-AND-APPLICATION-SHELL.md | 2 +- ...COVERY-AND-INBOUND-DELIVERY-RELIABILITY.md | 2 +- ...ASE-UPGRADE-AND-OPERATIONAL-RELIABILITY.md | 2 +- ...-RELIABILITY-AND-COMPACT-MAILBOX-READER.md | 2 +- ...Y-DATA-REPAIR-AND-RUNTIME-ERROR-CLOSURE.md | 47 ++- mailbox-app/LICENSES.md | 2 +- mailbox-app/apps/ingestion/parser.py | 372 +++++++++++++++++- mailbox-app/pyproject.toml | 5 +- mailbox-app/requirements/base.txt | 2 +- mailbox-app/requirements/constraints.txt | 2 +- mailbox-app/requirements/locked.txt | 3 +- mailbox-app/scripts/deploy_application.sh | 2 +- mailbox-app/scripts/preflight_v1_2_1.sh | 2 +- mailbox-app/scripts/verify_v1_2_1.sh | 2 +- .../tests/fixtures/html_inline_style.eml | 9 + .../tests/fixtures/html_malformed_css.eml | 14 + .../tests/fixtures/html_media_style.eml | 14 + .../tests/fixtures/html_style_safe.eml | 18 + .../tests/fixtures/html_style_unsafe.eml | 14 + .../tests/fixtures/html_table_layout.eml | 11 + .../tests/integration/test_auth_views.py | 9 + .../integration/test_message_body_repair.py | 7 +- .../tests/security/test_deployment_assets.py | 9 +- .../tests/security/test_security_controls.py | 29 ++ mailbox-app/tests/unit/test_parser_storage.py | 145 ++++++- 51 files changed, 981 insertions(+), 271 deletions(-) create mode 100644 mailbox-app/tests/fixtures/html_inline_style.eml create mode 100644 mailbox-app/tests/fixtures/html_malformed_css.eml create mode 100644 mailbox-app/tests/fixtures/html_media_style.eml create mode 100644 mailbox-app/tests/fixtures/html_style_safe.eml create mode 100644 mailbox-app/tests/fixtures/html_style_unsafe.eml create mode 100644 mailbox-app/tests/fixtures/html_table_layout.eml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d6e8a0b..b17ce99 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -131,5 +131,5 @@ jobs: - name: Release verification run: | python scripts/verify_release.py \ - dist/mailstack-1.3.4-rc.1-source.zip \ - --checksum dist/mailstack-1.3.4-rc.1-source.zip.sha256 + dist/mailstack-1.3.4-rc.2-source.zip \ + --checksum dist/mailstack-1.3.4-rc.2-source.zip.sha256 diff --git a/CHANGELOG.md b/CHANGELOG.md index bf7fe87..b4108e8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,10 @@ All notable repository-level changes are recorded here. Application history befo ### Corrected +- Advanced the pinned Django 5.2 LTS runtime from 5.2.16 to 5.2.17 after RC2 `pip-audit` identified the upstream security advisory, and synchronized active dependency, deployment verification, and security-test contracts without changing application behavior. +- Renamed CSS parser token-kind locals to avoid Bandit B105 credential-name false positives; no Bandit rule, exclusion, sanitizer policy, or runtime behavior is weakened. +- Added the RC2 high-fidelity reader candidate: safe inline CSS and bounded sanitized `") + return + if self._head_depth: + if lowered == "head": + self._head_depth -= 1 + return + if lowered == "head": + return self.output.append(f"") def handle_data(self, data: str) -> None: - if not self._suppressed_depth: + if self._style_depth: + self._style_buffer.append(data) + elif not self._suppressed_depth and not self._head_depth: self.output.append(data) def handle_entityref(self, name: str) -> None: - if not self._suppressed_depth: + if self._style_depth: + self._style_buffer.append(f"&{name};") + elif not self._suppressed_depth and not self._head_depth: self.output.append(f"&{name};") def handle_charref(self, name: str) -> None: - if not self._suppressed_depth: + if self._style_depth: + self._style_buffer.append(f"&#{name};") + elif not self._suppressed_depth and not self._head_depth: self.output.append(f"&#{name};") -def _prepare_html_for_sanitizer(value: str) -> str: - parser = _SanitizerPreprocessor() +def _prepare_html_for_sanitizer(value: str, css_sanitizer: _MailStackCSSSanitizer) -> str: + parser = _SanitizerPreprocessor(css_sanitizer) parser.feed(value or "") parser.close() return "".join(parser.output) @@ -185,7 +534,7 @@ def safe_filename(value: str | None) -> str: def _safe_attribute(tag: str, name: str, value: str) -> bool: lowered = name.lower() - if lowered.startswith("on") or lowered == "style": + if lowered.startswith("on"): return False if tag == "img" and lowered == "src": return value.lower().startswith(SAFE_DATA_IMAGE_PREFIXES) @@ -196,17 +545,20 @@ def _safe_attribute(tag: str, name: str, value: str) -> bool: def sanitize_html(value: str) -> str: + css_sanitizer = _MailStackCSSSanitizer(_CSSBudget()) cleaner = bleach.Cleaner( tags=SAFE_TAGS, attributes=_safe_attribute, protocols={"http", "https", "mailto", "data"}, strip=True, strip_comments=True, + css_sanitizer=css_sanitizer, ) - prepared = _prepare_html_for_sanitizer(value) + prepared = _prepare_html_for_sanitizer(value, css_sanitizer) cleaned = cleaner.clean(prepared) return bleach.linkifier.Linker( callbacks=[bleach.callbacks.nofollow, bleach.callbacks.target_blank], + skip_tags={"style"}, parse_email=False, ).linkify(cleaned).strip() diff --git a/mailbox-app/pyproject.toml b/mailbox-app/pyproject.toml index 24a8181..94278e9 100644 --- a/mailbox-app/pyproject.toml +++ b/mailbox-app/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "mailstack" -version = "1.3.4rc1" +version = "1.3.4rc2" description = "MailStack self-hosted receive-only mail server and shared team inbox" authors = [{ name = "Vib Tools" }] maintainers = [{ name = "Vib Tools" }] @@ -27,9 +27,10 @@ requires-python = ">=3.12,<3.13" license = "AGPL-3.0-or-later" readme = "README.md" dependencies = [ - "Django==5.2.16", + "Django==5.2.17", "argon2-cffi==25.1.0", "bleach==6.4.0", + "tinycss2==1.5.1", "filelock==3.20.3", "gunicorn==25.1.0", "mysqlclient==2.2.7", diff --git a/mailbox-app/requirements/base.txt b/mailbox-app/requirements/base.txt index 2b8238d..5aa92e1 100644 --- a/mailbox-app/requirements/base.txt +++ b/mailbox-app/requirements/base.txt @@ -1,4 +1,4 @@ -Django==5.2.16 +Django==5.2.17 argon2-cffi==25.1.0 bleach==6.4.0 filelock==3.20.3 diff --git a/mailbox-app/requirements/constraints.txt b/mailbox-app/requirements/constraints.txt index 1fe0a5f..c8bb3fe 100644 --- a/mailbox-app/requirements/constraints.txt +++ b/mailbox-app/requirements/constraints.txt @@ -6,7 +6,7 @@ bandit==1.9.4 bleach==6.4.0 cffi==2.0.0 coverage==7.13.3 -Django==5.2.16 +Django==5.2.17 filelock==3.20.3 gunicorn==25.1.0 packaging==26.2 diff --git a/mailbox-app/requirements/locked.txt b/mailbox-app/requirements/locked.txt index 83bf6b9..8cf762b 100644 --- a/mailbox-app/requirements/locked.txt +++ b/mailbox-app/requirements/locked.txt @@ -4,13 +4,14 @@ argon2-cffi-bindings==25.1.0 asgiref==3.11.1 bleach==6.4.0 cffi==2.0.0 -Django==5.2.16 +Django==5.2.17 filelock==3.20.3 gunicorn==25.1.0 packaging==26.2 pycparser==3.0 python-dotenv==1.2.2 sqlparse==0.6.0 +tinycss2==1.5.1 webencodings==0.5.1 whitenoise==6.11.0 diff --git a/mailbox-app/scripts/deploy_application.sh b/mailbox-app/scripts/deploy_application.sh index 0b9c159..04e3180 100755 --- a/mailbox-app/scripts/deploy_application.sh +++ b/mailbox-app/scripts/deploy_application.sh @@ -67,7 +67,7 @@ runuser -u vmail -- "$VENV/bin/python" - <<'PY' from importlib.metadata import version required = { - "Django": "5.2.16", + "Django": "5.2.17", "bleach": "6.4.0", "python-dotenv": "1.2.2", } diff --git a/mailbox-app/scripts/preflight_v1_2_1.sh b/mailbox-app/scripts/preflight_v1_2_1.sh index 1f1c721..9cc921a 100755 --- a/mailbox-app/scripts/preflight_v1_2_1.sh +++ b/mailbox-app/scripts/preflight_v1_2_1.sh @@ -14,7 +14,7 @@ case "$SOURCE" in esac [[ -f "$SOURCE/manage.py" && -f "$SOURCE/pyproject.toml" ]] || { printf 'Invalid release source.\n' >&2; exit 1; } grep -q '^version = "1.2.1"$' "$SOURCE/pyproject.toml" || { printf 'Release is not version 1.2.1.\n' >&2; exit 1; } -grep -q '^Django==5\.2\.16$' "$SOURCE/requirements/locked.txt" || { printf 'Required Django 5.2.16 security pin is missing.\n' >&2; exit 1; } +grep -q '^Django==5\.2\.17$' "$SOURCE/requirements/locked.txt" || { printf 'Required Django 5.2.17 security pin is missing.\n' >&2; exit 1; } grep -q '^bleach==6\.4\.0$' "$SOURCE/requirements/locked.txt" || { printf 'Required Bleach 6.4.0 security pin is missing.\n' >&2; exit 1; } grep -q '^python-dotenv==1\.2\.2$' "$SOURCE/requirements/locked.txt" || { printf 'Required python-dotenv 1.2.2 security pin is missing.\n' >&2; exit 1; } [[ -f "$SOURCE/RELEASE_MANIFEST.json" ]] || { printf 'Release manifest is missing.\n' >&2; exit 1; } diff --git a/mailbox-app/scripts/verify_v1_2_1.sh b/mailbox-app/scripts/verify_v1_2_1.sh index df8924b..219d26c 100755 --- a/mailbox-app/scripts/verify_v1_2_1.sh +++ b/mailbox-app/scripts/verify_v1_2_1.sh @@ -27,7 +27,7 @@ run_app "$PYTHON" - <<'PY' from importlib.metadata import version required = { - "Django": "5.2.16", + "Django": "5.2.17", "bleach": "6.4.0", "python-dotenv": "1.2.2", } diff --git a/mailbox-app/tests/fixtures/html_inline_style.eml b/mailbox-app/tests/fixtures/html_inline_style.eml new file mode 100644 index 0000000..47968d3 --- /dev/null +++ b/mailbox-app/tests/fixtures/html_inline_style.eml @@ -0,0 +1,9 @@ +From: Sender +To: mailbox1@example.com +Date: Tue, 18 Aug 2026 11:02:00 +0000 +Subject: Inline style +Message-ID: +MIME-Version: 1.0 +Content-Type: text/html; charset=utf-8 + +

Inline styled content.

diff --git a/mailbox-app/tests/fixtures/html_malformed_css.eml b/mailbox-app/tests/fixtures/html_malformed_css.eml new file mode 100644 index 0000000..a6588df --- /dev/null +++ b/mailbox-app/tests/fixtures/html_malformed_css.eml @@ -0,0 +1,14 @@ +From: Sender +To: mailbox1@example.com +Date: Tue, 18 Aug 2026 11:05:00 +0000 +Subject: Malformed CSS +Message-ID: +MIME-Version: 1.0 +Content-Type: text/html; charset=utf-8 + + +

Malformed CSS must not break the message.

diff --git a/mailbox-app/tests/fixtures/html_media_style.eml b/mailbox-app/tests/fixtures/html_media_style.eml new file mode 100644 index 0000000..40aa0da --- /dev/null +++ b/mailbox-app/tests/fixtures/html_media_style.eml @@ -0,0 +1,14 @@ +From: Sender +To: mailbox1@example.com +Date: Tue, 18 Aug 2026 11:03:00 +0000 +Subject: Media style +Message-ID: +MIME-Version: 1.0 +Content-Type: text/html; charset=utf-8 + + +
Responsive content.
diff --git a/mailbox-app/tests/fixtures/html_style_safe.eml b/mailbox-app/tests/fixtures/html_style_safe.eml new file mode 100644 index 0000000..2aec011 --- /dev/null +++ b/mailbox-app/tests/fixtures/html_style_safe.eml @@ -0,0 +1,18 @@ +From: Sender +To: mailbox1@example.com +Date: Tue, 18 Aug 2026 11:00:00 +0000 +Subject: Safe style block +Message-ID: +MIME-Version: 1.0 +Content-Type: text/html; charset=utf-8 + + + + + + +

Safe styled content

+ diff --git a/mailbox-app/tests/fixtures/html_style_unsafe.eml b/mailbox-app/tests/fixtures/html_style_unsafe.eml new file mode 100644 index 0000000..8d9f62c --- /dev/null +++ b/mailbox-app/tests/fixtures/html_style_unsafe.eml @@ -0,0 +1,14 @@ +From: Sender +To: mailbox1@example.com +Date: Tue, 18 Aug 2026 11:01:00 +0000 +Subject: Unsafe style block +Message-ID: +MIME-Version: 1.0 +Content-Type: text/html; charset=utf-8 + + +

Unsafe CSS filtered, content retained.

diff --git a/mailbox-app/tests/fixtures/html_table_layout.eml b/mailbox-app/tests/fixtures/html_table_layout.eml new file mode 100644 index 0000000..088de73 --- /dev/null +++ b/mailbox-app/tests/fixtures/html_table_layout.eml @@ -0,0 +1,11 @@ +From: Sender +To: mailbox1@example.com +Date: Tue, 18 Aug 2026 11:04:00 +0000 +Subject: Table layout +Message-ID: +MIME-Version: 1.0 +Content-Type: text/html; charset=utf-8 + + + +
Table cell.
diff --git a/mailbox-app/tests/integration/test_auth_views.py b/mailbox-app/tests/integration/test_auth_views.py index 9a549de..deede4c 100644 --- a/mailbox-app/tests/integration/test_auth_views.py +++ b/mailbox-app/tests/integration/test_auth_views.py @@ -158,11 +158,20 @@ def test_inbox_search_filters_detail_state_and_safe_html(client, admin_user, mai assert b'sandbox=""' in response.content assert AuditLog.objects.filter(action="message_view").exists() + message.sanitized_html_body = ( + '

Safe body

' + ) + message.save(update_fields=["sanitized_html_body", "updated_at"]) html_response = client.get(reverse("messages:safe_html", args=[message.uuid])) assert html_response.status_code == 200 assert b"Safe body" in html_response.content + assert b".brand{color:red;padding:8px;}" in html_response.content assert b"Remote content" not in html_response.content assert "default-src 'none'" in html_response["Content-Security-Policy"] + assert "img-src data:" in html_response["Content-Security-Policy"] + assert "style-src 'unsafe-inline'" in html_response["Content-Security-Policy"] + assert "form-action 'none'" in html_response["Content-Security-Policy"] + assert "navigate-to 'none'" in html_response["Content-Security-Policy"] assert html_response["X-Frame-Options"] == "SAMEORIGIN" message.refresh_from_db() diff --git a/mailbox-app/tests/integration/test_message_body_repair.py b/mailbox-app/tests/integration/test_message_body_repair.py index 373a7de..4b73817 100644 --- a/mailbox-app/tests/integration/test_message_body_repair.py +++ b/mailbox-app/tests/integration/test_message_body_repair.py @@ -67,7 +67,12 @@ def test_repair_message_bodies_dry_run_then_mutation_preserves_state(mailbox, fi message.refresh_from_db() assert "Welcome to Harpoon!" in message.sanitized_html_body - assert "#outlook" not in message.sanitized_html_body + lowered = message.sanitized_html_body.lower() + assert "" + '

' + "Safe

" + ).lower() + assert ".safe{color:red;padding:8px;}" in cleaned + assert "@media screen and (max-width:600px)" in cleaned + assert "font-weight:700;" in cleaned + assert "tracker.test" not in cleaned + assert "@import" not in cleaned + assert "background-image" not in cleaned + assert "position:fixed" not in cleaned + assert "transform" not in cleaned + assert "expression(" not in cleaned + assert "var(" not in cleaned + assert "--mail-width" not in cleaned + assert "data:image" not in cleaned + + @pytest.mark.django_db def test_stored_and_reflected_xss_are_escaped(client, admin_user, mailbox, message): message.subject = '' diff --git a/mailbox-app/tests/unit/test_parser_storage.py b/mailbox-app/tests/unit/test_parser_storage.py index 13c6022..d721c08 100644 --- a/mailbox-app/tests/unit/test_parser_storage.py +++ b/mailbox-app/tests/unit/test_parser_storage.py @@ -1,6 +1,7 @@ from __future__ import annotations import hashlib +import re from pathlib import Path import pytest @@ -68,19 +69,100 @@ def test_sanitize_html_removes_active_and_remote_content(): assert "Text" in cleaned -def test_sanitize_html_drops_style_block_text_and_remote_image_nodes(): +def test_sanitize_html_preserves_safe_style_block_without_visible_css_leak(): cleaned = sanitize_html( "" "

Readable email content

" "intercom" ) lowered = cleaned.lower() - assert "#outlook" not in lowered - assert ".readmsgbody" not in lowered + assert "", "", lowered, flags=re.DOTALL) + assert "#outlook" not in visible_markup + assert ".readmsgbody" not in visible_markup + + +def test_sanitize_html_keeps_only_sanitized_style_from_head(): + cleaned = sanitize_html( + "HEAD SECRET" + "Hidden title" + "" + "" + "" + "

Visible body

" + ) + lowered = cleaned.lower() + assert "head secret" not in lowered + assert "hidden title" not in lowered + assert "description" not in lowered + assert "alert(" not in lowered + assert "tracker.test" not in lowered + assert ".brand{color:#123456;}" in lowered + assert "Visible body" in cleaned + + +def test_sanitize_html_preserves_safe_inline_css_and_drops_unsafe_declarations(): + cleaned = sanitize_html( + '

Styled

' + ) + lowered = cleaned.lower() + assert "color:red;" in lowered + assert "padding:12px;" in lowered + assert "width:calc(100% - 20px);" in lowered + assert "background-image" not in lowered + assert "tracker.test" not in lowered + assert "position:fixed" not in lowered + + +def test_sanitize_html_allows_bounded_media_rules_and_drops_other_at_rules(): + cleaned = sanitize_html( + "
Responsive
" + ) + lowered = cleaned.lower() + assert "@media screen and (max-width:600px)" in lowered + assert ".responsive{width:100%;display:block;}" in lowered + assert "@import" not in lowered + assert "@font-face" not in lowered + assert "prefers-color-scheme" not in lowered + assert "tracker.test" not in lowered + + +def test_sanitize_html_css_fail_closed_limits_do_not_drop_message_content(): + oversized = "color:red;" * 20_000 + cleaned = sanitize_html(f'

Still readable

') + assert "Still readable" in cleaned + assert 'style=""' in cleaned or "style" not in cleaned + + oversized_stylesheet = ".safe{color:red;}" * 10_000 + cleaned = sanitize_html(f"

Body survives

") + assert "Body survives" in cleaned + assert "", "", lowered, flags=re.DOTALL) + assert "#outlook" not in visible_markup + assert ".readmsgbody" not in visible_markup + + +def test_parse_message_css_fixture_contract(fixtures_dir: Path): + safe = parse_message((fixtures_dir / "html_style_safe.eml").read_bytes()).sanitized_html_body.lower() + unsafe = parse_message((fixtures_dir / "html_style_unsafe.eml").read_bytes()).sanitized_html_body.lower() + inline = parse_message((fixtures_dir / "html_inline_style.eml").read_bytes()).sanitized_html_body.lower() + media = parse_message((fixtures_dir / "html_media_style.eml").read_bytes()).sanitized_html_body.lower() + table = parse_message((fixtures_dir / "html_table_layout.eml").read_bytes()).sanitized_html_body.lower() + malformed = parse_message( + (fixtures_dir / "html_malformed_css.eml").read_bytes() + ).sanitized_html_body.lower() + + assert ".brand{color:#123456;font-size:18px;padding:12px;}" in safe + assert "font-weight:700 !important;" in safe + + assert ".safe{color:blue;}" in unsafe + assert "@import" not in unsafe + assert "@font-face" not in unsafe + assert "tracker.example.test" not in unsafe + assert "position:fixed" not in unsafe + assert "z-index" not in unsafe + + assert "color:rgb(10, 20, 30);" in inline + assert "padding:16px;" in inline + assert "width:calc(100% - 20px);" in inline + assert "background-image" not in inline + assert "position:fixed" not in inline + + assert "@media screen and (max-width: 600px)" in media + assert "prefers-color-scheme" not in media + assert "@media only" not in media + + assert "width:600px;" in table + assert "border-collapse:collapse;" in table + assert "table-layout:fixed;" in table + assert "background-color:#f4f4f4;" in table + assert "vertical-align:top;" in table + + assert "malformed css must not break the message." in malformed + assert "padding:10px;" in malformed + assert "expression(" not in malformed def test_parse_message_extracts_headers_bodies_and_attachments(fixtures_dir: Path):