Fix admin crashes and Resonate worker going idle

backend/conferences/admin/conference.py

@@ -49,6 +49,9 @@ def validate_deadlines_form(forms):
         if not form.cleaned_data:
             return
 
+        if form.errors:
+            continue
+
         start = form.cleaned_data["start"]
         end = form.cleaned_data["end"]
         delete = form.cleaned_data["DELETE"]
@@ -74,6 +77,12 @@ class Meta:
         model = Deadline
         fields = ["start", "end", "name", "description", "type", "conference"]
 
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        # TimeFramedModel leaves both dates optional; a deadline needs them.
+        self.fields["start"].required = True
+        self.fields["end"].required = True
+
 
 class DeadlineFormSet(BaseInlineFormSet):
     def clean(self):

backend/conferences/tests/test_admin.py

@@ -154,6 +154,33 @@ def test_start_date_comes_before_end(add_delete_field_to_form):
         validate_deadlines_form(forms)
 
 
+@mark.parametrize(
+    "field,value", [("start", "not a date"), ("end", ""), ("type", "unknown")]
+)
+def test_invalid_deadline_field_is_left_to_its_field_error(
+    field, value, add_delete_field_to_form
+):
+    conference = ConferenceFactory()
+    data = {
+        "start": "2021-11-10 01:43:58",
+        "end": "2022-11-10 01:43:58",
+        "name_0": "en",
+        "name_1": "it",
+        "description_0": "descen",
+        "description_1": "descit",
+        "type": "cfp",
+        "conference": conference.id,
+        "DELETE": False,
+    }
+    data[field] = value
+    form = DeadlineForm(data=data)
+    form.is_valid()
+
+    validate_deadlines_form([form])
+
+    assert field in form.errors
+
+
 def test_can_have_as_many_custom_deadlines_as_we_want(add_delete_field_to_form):
     conference = ConferenceFactory()
     form_1 = DeadlineForm(

backend/files_upload/admin.py

@@ -20,7 +20,7 @@ class FileAdmin(admin.ModelAdmin):
     list_display = ("id", "file", "created", "uploaded_by")
     search_fields = (
         "file",
-        "uploaded_by",
+        "uploaded_by__email",
         "id",
     )
     list_filter = ("created", "type", "virus", "mime_type")

backend/files_upload/tests/test_admin.py

@@ -0,0 +1,22 @@
+import pytest
+from django.contrib.admin.sites import AdminSite
+
+from files_upload.admin import FileAdmin
+from files_upload.models import File
+from files_upload.tests.factories import FileFactory
+from users.tests.factories import UserFactory
+
+pytestmark = pytest.mark.django_db
+
+
+def test_search_files_by_uploader_email(rf, admin_superuser):
+    file = FileFactory(uploaded_by=UserFactory(email="uploader@example.org"))
+    FileFactory(uploaded_by=UserFactory(email="someone@example.org"))
+    request = rf.get("/")
+    request.user = admin_superuser
+
+    results, _ = FileAdmin(File, AdminSite()).get_search_results(
+        request, File.objects.all(), "uploader@"
+    )
+
+    assert list(results) == [file]

backend/grants/admin.py

@@ -10,6 +10,7 @@
 from django.utils import timezone
 from django.utils.html import format_html, format_html_join
 from django.utils.safestring import mark_safe
+from requests import RequestException
 from import_export.admin import ExportMixin
 from import_export.fields import Field
 from import_export.resources import ModelResource
@@ -202,14 +203,22 @@ def _add_form_question_fields(self, conference_id):
             self.fields[key] = Field(column_name=question.label)
             self._question_fields[key] = question
 
-    def export_field(self, field, obj):
+    def export_field(self, field, instance, **kwargs):
         question = self._question_fields.get(self.get_field_name(field))
         if question is not None:
-            if obj.form_answer_id is None:
+            if instance.form_answer_id is None:
                 return ""
-            answers = unwrap_answers(obj.form_answer.answers)
+            answers = unwrap_answers(instance.form_answer.answers)
             return display_answer_value(question, answers.get(str(question.pk)))
-        return super().export_field(field, obj)
+        return super().export_field(field, instance, **kwargs)
+
+    def get_export_fields(self, selected_fields=None):
+        # The admin's export form lists fields from get_export_order(), which
+        # cannot know the per-conference question columns.
+        export_fields = super().get_export_fields(selected_fields)
+        if selected_fields:
+            export_fields += [self.fields[key] for key in self._question_fields]
+        return export_fields
 
     class Meta:
         model = Grant
@@ -376,11 +385,18 @@ def create_grant_vouchers(modeladmin, request, queryset):
         existing = existing_by_user.get(grant.user_id)
 
         if not existing:
-            new_voucher = create_conference_voucher(
-                conference=grant.conference,
-                user=grant.user,
-                voucher_type=ConferenceVoucher.VoucherType.GRANT,
-            )
+            try:
+                new_voucher = create_conference_voucher(
+                    conference=grant.conference,
+                    user=grant.user,
+                    voucher_type=ConferenceVoucher.VoucherType.GRANT,
+                )
+            except RequestException as e:
+                messages.error(
+                    request,
+                    f"Could not create a voucher for {grant.name} on Pretix: {e}",
+                )
+                continue
             existing_by_user[grant.user_id] = new_voucher
             create_addition_admin_log_entry(
                 request.user,

backend/grants/models.py

@@ -233,9 +233,11 @@ class GrantType(models.TextChoices):
 
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
-        self._original_status = self.status
-        self._original_pending_status = self.pending_status
-        self._original_country_type = self.country_type
+        # Read through __dict__: touching a deferred field here refetches the
+        # row, which builds another Grant and recurses.
+        self._original_status = self.__dict__.get("status")
+        self._original_pending_status = self.__dict__.get("pending_status")
+        self._original_country_type = self.__dict__.get("country_type")
 
     def __str__(self):
         return f"{self.full_name}"

backend/grants/tests/test_admin.py

@@ -6,6 +6,7 @@
 from django.contrib.admin.models import LogEntry
 from django.contrib.admin.sites import AdminSite
 from django.utils import timezone
+from requests import HTTPError
 
 from conferences.models.conference_voucher import ConferenceVoucher
 from conferences.tests.factories import ConferenceFactory, ConferenceVoucherFactory
@@ -313,6 +314,35 @@ def test_send_reply_email_waiting_list_update(rf, mocker, admin_user):
     ).exists()
 
 
+def test_create_grant_vouchers_reports_pretix_failures_and_continues(
+    rf, mocker, admin_user
+):
+    mock_messages = mocker.patch("grants.admin.messages")
+    mocker.patch(
+        "conferences.vouchers.create_voucher",
+        side_effect=[HTTPError("Pretix is down"), {"id": 999}],
+    )
+    conference = ConferenceFactory()
+    failed_grant = GrantFactory(status=Grant.Status.confirmed, conference=conference)
+    grant = GrantFactory(status=Grant.Status.confirmed, conference=conference)
+    request = rf.get("/")
+    request.user = admin_user
+
+    create_grant_vouchers(
+        None,
+        request=request,
+        queryset=Grant.objects.filter(conference=conference),
+    )
+
+    assert list(ConferenceVoucher.objects.values_list("user_id", flat=True)) == [
+        grant.user_id
+    ]
+    mock_messages.error.assert_called_once_with(
+        request,
+        f"Could not create a voucher for {failed_grant.name} on Pretix: Pretix is down",
+    )
+
+
 def test_create_grant_vouchers(rf, mocker, admin_user):
     mock_messages = mocker.patch("grants.admin.messages")
     mocker.patch("conferences.vouchers.create_voucher", return_value={"id": 999})
@@ -851,7 +881,8 @@ def test_dynamic_answers_has_an_empty_state_for_legacy_grants():
 
 def _export_grants(conference):
     queryset = Grant.objects.filter(conference=conference).order_by("id")
-    return GrantResource().export(queryset=queryset)
+    resource = GrantResource()
+    return resource.export(queryset=queryset, export_fields=resource.get_export_order())
 
 
 def test_export_includes_one_column_per_form_question():

backend/grants/tests/test_models.py

@@ -309,6 +309,15 @@ def test_sets_country_type_does_nothing_if_unset():
     assert grant.country_type is None
 
 
+def test_deleting_conference_deletes_its_grants():
+    grant = GrantFactory()
+    other_grant = GrantFactory()
+
+    grant.conference.delete()
+
+    assert list(Grant.objects.all()) == [other_grant]
+
+
 def test_pending_status_no_longer_syncs_with_status():
     grant = GrantFactory(
         pending_status=Grant.Status.pending,

backend/pycon/resonate_app.py

@@ -19,27 +19,54 @@
 import asyncio
 import functools
 import inspect
+import uuid
 from typing import Any, Callable, TypeVar, get_type_hints
 
+import aiohttp
 from asgiref.sync import sync_to_async
 from django.conf import settings
 from django.core.exceptions import ImproperlyConfigured
 from django.db import connections
 from django.utils.module_loading import autodiscover_modules
+from resonate.connections.sse import SSEConnection
 from resonate.resonate import Resonate
 
 _resonate: Resonate | None = None
 
+
+class WorkerPollConnection(SSEConnection):
+    # The SDK opens its long-lived poll stream with aiohttp's default 5 minute
+    # total timeout, and the bare TimeoutError escapes its reconnect loop.
+    def _ensure_session(self) -> aiohttp.ClientSession:
+        if self._session is None and not self._stopped:
+            self._session = aiohttp.ClientSession(
+                timeout=aiohttp.ClientTimeout(total=None, sock_connect=30)
+            )
+        return super()._ensure_session()
+
+
 T = TypeVar("T")
 
 
 def _build(*, send_only: bool) -> Resonate:
+    pid = uuid.uuid4().hex
+    sources: list[WorkerPollConnection] | None = None
+    if send_only:
+        # An empty list of sources says this process only creates promises and
+        # never picks work up.
+        sources = []
+    elif settings.RESONATE_URL:
+        sources = [
+            WorkerPollConnection(
+                url=settings.RESONATE_URL, pid=pid, group=settings.RESONATE_GROUP
+            )
+        ]
+
     return Resonate(
         group=settings.RESONATE_GROUP,
         url=settings.RESONATE_URL or None,
-        # An empty list of sources says this process only creates promises and
-        # never picks work up; ``None`` lets the SDK give it the usual one.
-        sources=[] if send_only else None,
+        pid=pid,
+        sources=sources,
         # Nothing here needs an event loop: connections are opened by
         # ``start()``, so building at import time is safe.
         autostart=False,

backend/pycon/tests/test_resonate_worker.py

@@ -1,8 +1,44 @@
+import asyncio
+
+import aiohttp
+from aiohttp import web
+from aiohttp.test_utils import TestServer
 from django.core.exceptions import ImproperlyConfigured
 from django.core.management import call_command
 from pytest import raises
 
 from pycon.management.commands.resonate_worker import Command
+from pycon.resonate_app import WorkerPollConnection
+
+
+def test_worker_keeps_receiving_work_past_aiohttp_default_timeout(mocker):
+    mocker.patch("aiohttp.client.DEFAULT_TIMEOUT", aiohttp.ClientTimeout(total=0.5))
+    received = []
+
+    async def stream(request):
+        response = web.StreamResponse(headers={"Content-Type": "text/event-stream"})
+        await response.prepare(request)
+        await response.write(b"data: first\n\n")
+        await asyncio.sleep(1)
+        await response.write(b"data: second\n\n")
+        await asyncio.sleep(5)
+        return response
+
+    async def listen():
+        app = web.Application()
+        app.router.add_get("/poll/{group}/{pid}", stream)
+        async with TestServer(app) as server:
+            connection = WorkerPollConnection(
+                url=str(server.make_url("")), group="pycon"
+            )
+            connection.recv(received.append)
+            await connection.start()
+            await asyncio.sleep(1.5)
+            await connection.stop()
+
+    asyncio.run(listen())
+
+    assert received == ["first", "second"]
 
 
 def test_worker_reloads_on_code_changes_by_default(mocker, settings):