From 474cd09d2c219885334baca25e731e6474976d2d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dean=20Qui=C3=B1anola?= Date: Fri, 13 Nov 2026 09:03:27 -0800 Subject: [PATCH 1/3] perf(serverless): take the next job while a result uploads --- runpod/serverless/modules/rp_job.py | 16 ++- runpod/serverless/modules/rp_scale.py | 59 ++++++-- tests/test_serverless/test_result_overlap.py | 138 +++++++++++++++++++ 3 files changed, 200 insertions(+), 13 deletions(-) create mode 100644 tests/test_serverless/test_result_overlap.py diff --git a/runpod/serverless/modules/rp_job.py b/runpod/serverless/modules/rp_job.py index 99addbc74..e780c6485 100644 --- a/runpod/serverless/modules/rp_job.py +++ b/runpod/serverless/modules/rp_job.py @@ -194,7 +194,18 @@ async def get_job( return jobs -async def handle_job(session: ClientSession, config: Dict[str, Any], job) -> dict: +async def handle_job( + session: ClientSession, + config: Dict[str, Any], + job, + on_result_ready: Optional[Callable[[Dict[str, Any]], None]] = None, +) -> dict: + """Run a job and send its result. + + on_result_ready, if given, is called with the final result just before it + is sent, so the caller can free the job's slot and take the next job + while the result upload is still in flight. + """ if is_generator(config["handler"]): is_stream = True generator_output = run_job_generator(config["handler"], job) @@ -237,6 +248,9 @@ async def handle_job(session: ClientSession, config: Dict[str, Any], job) -> dic log.debug("rp_debugger | Flag not set, skipping debugger output.", job["id"]) rp_debugger.clear_debugger_output() + if on_result_ready is not None: + on_result_ready(job_result) + # Send the job result back to JOB_DONE_URL await send_result(session, job_result, job, is_stream=is_stream) return job_result diff --git a/runpod/serverless/modules/rp_scale.py b/runpod/serverless/modules/rp_scale.py index 8537ebab2..7ed9eec51 100644 --- a/runpod/serverless/modules/rp_scale.py +++ b/runpod/serverless/modules/rp_scale.py @@ -80,6 +80,9 @@ def __init__(self, config: dict[str, Any]): # maps in-progress job ids to their running tasks so individual jobs # can be stopped without killing the whole worker self.jobs_tasks: dict[str, asyncio.Task[Any]] = {} + # Jobs whose handler finished and whose result is still being sent. + # They no longer hold a slot, so the next job-take overlaps the upload. + self._sending_results: set[str] = set() self.stop_signals_fetcher = get_stop_signals self.stop_signals_fetcher_timeout = 90 @@ -416,8 +419,12 @@ async def run_jobs(self, session: ClientSession): # Clear before draining: a job queued after this point re-arms it. self._jobs_available.clear() - # Fetch as many jobs as the concurrency allows - while len(tasks) < self.current_concurrency and not self.jobs_queue.empty(): + # Fetch as many jobs as the concurrency allows; a task that is only + # sending its result no longer counts against it + while ( + len(tasks) - len(self._sending_results) < self.current_concurrency + and not self.jobs_queue.empty() + ): job = await self.jobs_queue.get() # Create a new task for each job and track it by job id task = asyncio.create_task(self.handle_job(session, job)) @@ -562,15 +569,16 @@ async def handle_job(self, session: ClientSession, job: dict[str, Any]): ) return - job_result = await self.jobs_handler(session, self.config, job) - - # Stop intake before the finally frees this slot: the platform - # restarts the worker on stopPod, killing any job taken meanwhile. - if self.config.get("refresh_worker", False) or ( - isinstance(job_result, dict) and job_result.get("stopPod") - ): - log.info("Refresh requested, taking no more jobs.", job["id"]) - self.kill_worker() + if self.jobs_handler is handle_job: + job_result = await self.jobs_handler( + session, + self.config, + job, + on_result_ready=lambda result: self._release_slot(job, result), + ) + else: + job_result = await self.jobs_handler(session, self.config, job) + self._stop_intake_if_refreshing(job, job_result) except asyncio.CancelledError: log.info("Job stopped.", job["id"]) @@ -585,13 +593,40 @@ async def handle_job(self, session: ClientSession, job: dict[str, Any]): self.jobs_queue.task_done() # Job is no longer in progress - self.job_progress.remove(job) + if job["id"] in self._sending_results: + self._sending_results.discard(job["id"]) + else: + self.job_progress.remove(job) self.jobs_tasks.pop(job["id"], None) self._slot_freed.set() log.debug("Finished Job", job["id"]) _reset_batch_id(batch_id_token) + def _stop_intake_if_refreshing(self, job: dict[str, Any], job_result: Any) -> None: + """Stop intake before this job's slot is freed: the platform restarts + the worker on stopPod, killing any job taken meanwhile.""" + if not self.is_alive(): + return + if self.config.get("refresh_worker", False) or ( + isinstance(job_result, dict) and job_result.get("stopPod") + ): + log.info("Refresh requested, taking no more jobs.", job["id"]) + self.kill_worker() + + def _release_slot(self, job: dict[str, Any], job_result: Any) -> None: + """Free a finished job's slot before its result is sent. + + The result upload takes a full round trip (~0.3 s measured on + serverless), and a single-slot worker used to wait it out before + asking for the next job. Taking the next job while the upload is in + flight hides it for every job queued behind another. + """ + self._stop_intake_if_refreshing(job, job_result) + self._sending_results.add(job["id"]) + self.job_progress.remove(job) + self._slot_freed.set() + async def _wait_for_prestart(self) -> None: """Wait for prestart to finish or shutdown to begin.""" ready = asyncio.create_task(self._prestart_ready.wait()) diff --git a/tests/test_serverless/test_result_overlap.py b/tests/test_serverless/test_result_overlap.py new file mode 100644 index 000000000..51fb4ed1a --- /dev/null +++ b/tests/test_serverless/test_result_overlap.py @@ -0,0 +1,138 @@ +""" +A finished job frees its slot before its result is sent, so the next job-take +overlaps the result upload instead of waiting for it. + +The upload is a full round trip to ai-api (~0.3 s measured on serverless), and +a single-slot worker used to wait it out before asking for the next job: every +job queued behind another paid it again. The refresh guarantee (SLS-900) still +holds: a job asking for a refresh stops intake before its slot is freed. +""" + +import asyncio +import time + +import pytest + +from runpod.serverless.modules import rp_http, rp_job, rp_scale + +from .test_refresh_worker import DEADLINE, FakeAiApi, _sync_handler, generate_job + +UPLOAD_S = 0.3 + + +class SlowUploadAiApi(FakeAiApi): + """ai-api whose job-done answers only after UPLOAD_S, recording when each + job-take arrived and when each upload finished.""" + + def __init__(self, jobs): + super().__init__(jobs) + self.take_at: dict[str, float] = {} + self.done_at: dict[str, float] = {} + + async def job_take(self, request): + response = await super().job_take(request) + if self.takes and self.takes[-1] is not None: + self.take_at.setdefault(self.takes[-1], time.monotonic()) + return response + + async def job_done(self, request): + await asyncio.sleep(UPLOAD_S) + response = await super().job_done(request) + self.done_at[request.match_info["job_id"]] = time.monotonic() + return response + + +@pytest.fixture +async def slow_upload_worker(monkeypatch): + servers = [] + + async def make(jobs): + server = SlowUploadAiApi(jobs) + await server.start() + servers.append(server) + monkeypatch.setattr(rp_job, "JOB_GET_URL", server.job_take_url.replace("$ID", "pod-1")) + monkeypatch.setattr( + rp_http, "JOB_DONE_URL", server.job_done_url.replace("$RUNPOD_POD_ID", "pod-1") + ) + return server + + yield make + for server in servers: + await server.close() + + +async def _serve_until_drained(server, scaler): + run = asyncio.create_task(scaler.run()) + try: + await asyncio.wait_for(server.polled_empty.wait(), timeout=DEADLINE) + # the last upload may still be in flight when the queue reads empty + deadline = time.monotonic() + DEADLINE + while len(server.done) < len(server.take_at) and time.monotonic() < deadline: + await asyncio.sleep(0.01) + finally: + scaler.kill_worker() + run.cancel() + await asyncio.gather(run, return_exceptions=True) + + +@pytest.mark.asyncio +async def test_next_job_is_taken_while_the_previous_result_uploads(slow_upload_worker): + server = await slow_upload_worker([generate_job(j, n=i) for i, j in enumerate("ABC")]) + await _serve_until_drained(server, rp_scale.JobScaler({"handler": _sync_handler})) + + assert server.takes[:3] == ["A", "B", "C"] + assert server.done == {j: {"output": {"echo": i}} for i, j in enumerate("ABC")} + assert server.take_at["B"] < server.done_at["A"] + assert server.take_at["C"] < server.done_at["B"] + + +@pytest.mark.asyncio +async def test_uploads_no_longer_add_up_across_queued_jobs(slow_upload_worker): + jobs = [generate_job(f"J{i}", n=i) for i in range(5)] + server = await slow_upload_worker(jobs) + await _serve_until_drained(server, rp_scale.JobScaler({"handler": _sync_handler})) + + elapsed = server.done_at["J4"] - server.take_at["J0"] + # serial: 5 uploads back to back (1.5 s); overlapped: about one upload + # plus the takes, which this fake answers at once + assert elapsed < 3 * UPLOAD_S, f"uploads were serialized: {elapsed:.2f}s" + + +@pytest.mark.asyncio +async def test_refresh_still_leaves_the_next_job_queued(slow_upload_worker): + server = await slow_upload_worker( + [generate_job("A", n=1, refresh=True), generate_job("B", n=2)] + ) + scaler = rp_scale.JobScaler({"handler": _sync_handler}) + run = asyncio.create_task(scaler.run()) + try: + _, pending = await asyncio.wait({run}, timeout=DEADLINE) + assert server.takes == ["A"], "job B was taken after stopPod and would be lost" + assert [job["id"] for job in server.queue] == ["B"] + assert server.done == {"A": {"output": {"echo": 1}, "stopPod": True}} + assert not pending + finally: + scaler.kill_worker() + run.cancel() + await asyncio.gather(run, return_exceptions=True) + + +@pytest.mark.asyncio +async def test_custom_jobs_handler_keeps_the_slot_until_it_returns(): + """Local-test jobs handlers have no on_result_ready; they keep the old contract.""" + seen = [] + + async def handler(session, config, job): + seen.append(job["id"]) + return {"output": job["id"]} + + scaler = rp_scale.JobScaler({"handler": _sync_handler}) + scaler.jobs_handler = handler + job = generate_job("A") + scaler.job_progress.add(job) + await scaler.jobs_queue.put(job) + await scaler.jobs_queue.get() + await scaler.handle_job(None, job) + assert seen == ["A"] + assert scaler.current_occupancy() == 0 + assert scaler._sending_results == set() From c8855971ffd4a561c0458790a9d6f37ff17a89d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dean=20Qui=C3=B1anola?= Date: Mon, 16 Nov 2026 10:55:20 -0800 Subject: [PATCH 2/3] perf(serverless): bound and order overlapping result uploads Address review on the result-upload overlap: - cap concurrent uploads (MAX_PENDING_RESULT_UPLOADS) so a slow result endpoint cannot retain unbounded results or open unbounded connections - make set_scale wait for in-flight uploads before swapping jobs_queue - await earlier uploads before a refresh result is sent, so a worker restart cannot drop a result still in flight --- runpod/serverless/modules/rp_job.py | 13 +- runpod/serverless/modules/rp_scale.py | 47 +++++- tests/test_serverless/test_result_overlap.py | 146 ++++++++++++++++++- 3 files changed, 194 insertions(+), 12 deletions(-) diff --git a/runpod/serverless/modules/rp_job.py b/runpod/serverless/modules/rp_job.py index e780c6485..91241178c 100644 --- a/runpod/serverless/modules/rp_job.py +++ b/runpod/serverless/modules/rp_job.py @@ -6,7 +6,7 @@ import json import os import traceback -from typing import Any, AsyncGenerator, Callable, Dict, Optional, Union, List +from typing import Any, AsyncGenerator, Awaitable, Callable, Dict, Optional, Union, List import aiohttp @@ -198,13 +198,16 @@ async def handle_job( session: ClientSession, config: Dict[str, Any], job, - on_result_ready: Optional[Callable[[Dict[str, Any]], None]] = None, + on_result_ready: Optional[ + Callable[[Dict[str, Any]], Optional[Awaitable[None]]] + ] = None, ) -> dict: """Run a job and send its result. on_result_ready, if given, is called with the final result just before it is sent, so the caller can free the job's slot and take the next job - while the result upload is still in flight. + while the result upload is still in flight. It may return an awaitable, + which is awaited before the result is sent. """ if is_generator(config["handler"]): is_stream = True @@ -249,7 +252,9 @@ async def handle_job( rp_debugger.clear_debugger_output() if on_result_ready is not None: - on_result_ready(job_result) + maybe_awaitable = on_result_ready(job_result) + if inspect.isawaitable(maybe_awaitable): + await maybe_awaitable # Send the job result back to JOB_DONE_URL await send_result(session, job_result, job, is_stream=is_stream) diff --git a/runpod/serverless/modules/rp_scale.py b/runpod/serverless/modules/rp_scale.py index 7ed9eec51..a486aae97 100644 --- a/runpod/serverless/modules/rp_scale.py +++ b/runpod/serverless/modules/rp_scale.py @@ -32,6 +32,13 @@ # wait so one long job can't hold the other slots idle, and so set_scale still # runs as often as it did under the old fixed sleep. FULL_WAIT_SECONDS = 1 +# Cap the result uploads that may overlap new work. The shared HTTP client sets +# no connection limit (TCPConnector(limit=0)) and a 600 s timeout, so without a +# ceiling a stalled result endpoint would let a worker retain unbounded results +# and open unbounded connections while it keeps taking jobs. Normal pipelines +# stay well under this; it only applies backpressure to a persistently slow +# result endpoint. +MAX_PENDING_RESULT_UPLOADS = 50 def _handle_uncaught_exception(exc_type, exc_value, exc_traceback): @@ -127,8 +134,9 @@ async def set_scale(self): # no need to resize return - while self.current_occupancy() > 0: - # not safe to scale when jobs are in flight + while self.current_occupancy() > 0 or self._sending_results: + # not safe to scale while jobs are in flight or results are still + # uploading: a swapped queue would receive the upload's task_done() await asyncio.sleep(1) continue @@ -420,9 +428,12 @@ async def run_jobs(self, session: ClientSession): self._jobs_available.clear() # Fetch as many jobs as the concurrency allows; a task that is only - # sending its result no longer counts against it + # sending its result no longer counts against it. Stop once too many + # uploads are already in flight so a slow result endpoint cannot + # accumulate unbounded tasks, connections, and retained results. while ( len(tasks) - len(self._sending_results) < self.current_concurrency + and len(self._sending_results) < MAX_PENDING_RESULT_UPLOADS and not self.jobs_queue.empty() ): job = await self.jobs_queue.get() @@ -603,18 +614,23 @@ async def handle_job(self, session: ClientSession, job: dict[str, Any]): log.debug("Finished Job", job["id"]) _reset_batch_id(batch_id_token) + def _requests_refresh(self, job_result: Any) -> bool: + """Whether this job asks the worker to refresh (restart) after it.""" + return bool( + self.config.get("refresh_worker", False) + or (isinstance(job_result, dict) and job_result.get("stopPod")) + ) + def _stop_intake_if_refreshing(self, job: dict[str, Any], job_result: Any) -> None: """Stop intake before this job's slot is freed: the platform restarts the worker on stopPod, killing any job taken meanwhile.""" if not self.is_alive(): return - if self.config.get("refresh_worker", False) or ( - isinstance(job_result, dict) and job_result.get("stopPod") - ): + if self._requests_refresh(job_result): log.info("Refresh requested, taking no more jobs.", job["id"]) self.kill_worker() - def _release_slot(self, job: dict[str, Any], job_result: Any) -> None: + async def _release_slot(self, job: dict[str, Any], job_result: Any) -> None: """Free a finished job's slot before its result is sent. The result upload takes a full round trip (~0.3 s measured on @@ -622,10 +638,27 @@ def _release_slot(self, job: dict[str, Any], job_result: Any) -> None: asking for the next job. Taking the next job while the upload is in flight hides it for every job queued behind another. """ + refreshing = self._requests_refresh(job_result) self._stop_intake_if_refreshing(job, job_result) self._sending_results.add(job["id"]) self.job_progress.remove(job) self._slot_freed.set() + if refreshing: + # Let earlier uploads land before this refresh result reaches the + # platform: once it sees stopPod it can restart the worker and kill + # a result still in flight. + await self._drain_earlier_uploads(job["id"]) + + async def _drain_earlier_uploads(self, job_id: str) -> None: + """Wait for every in-flight result upload other than this job's.""" + pending = [ + task + for other_id in list(self._sending_results) + if other_id != job_id + and (task := self.jobs_tasks.get(other_id)) is not None + ] + if pending: + await asyncio.gather(*pending, return_exceptions=True) async def _wait_for_prestart(self) -> None: """Wait for prestart to finish or shutdown to begin.""" diff --git a/tests/test_serverless/test_result_overlap.py b/tests/test_serverless/test_result_overlap.py index 51fb4ed1a..5bf2722ae 100644 --- a/tests/test_serverless/test_result_overlap.py +++ b/tests/test_serverless/test_result_overlap.py @@ -15,7 +15,13 @@ from runpod.serverless.modules import rp_http, rp_job, rp_scale -from .test_refresh_worker import DEADLINE, FakeAiApi, _sync_handler, generate_job +from .test_refresh_worker import ( + DEADLINE, + DummyProgress, + FakeAiApi, + _sync_handler, + generate_job, +) UPLOAD_S = 0.3 @@ -117,6 +123,144 @@ async def test_refresh_still_leaves_the_next_job_queued(slow_upload_worker): await asyncio.gather(run, return_exceptions=True) +class PerJobUploadAiApi(FakeAiApi): + """ai-api that delays job-done only for named jobs and records the order in + which results are received.""" + + def __init__(self, jobs, slow_jobs, delay=UPLOAD_S): + super().__init__(jobs) + self.slow_jobs = set(slow_jobs) + self.delay = delay + self.done_order: list[str] = [] + + async def job_done(self, request): + job_id = request.match_info["job_id"] + if job_id in self.slow_jobs: + await asyncio.sleep(self.delay) + response = await super().job_done(request) + self.done_order.append(job_id) + return response + + +class BlockingUploadAiApi(FakeAiApi): + """ai-api whose job-done blocks until released, so a test can watch how many + uploads the worker keeps in flight at once.""" + + def __init__(self, jobs): + super().__init__(jobs) + self.release = asyncio.Event() + self.uploading = 0 + self.max_uploading = 0 + + async def job_done(self, request): + self.uploading += 1 + self.max_uploading = max(self.max_uploading, self.uploading) + try: + await self.release.wait() + finally: + self.uploading -= 1 + return await super().job_done(request) + + +def _point_sdk_at(monkeypatch, server): + monkeypatch.setattr( + rp_job, "JOB_GET_URL", server.job_take_url.replace("$ID", "pod-1") + ) + monkeypatch.setattr( + rp_http, "JOB_DONE_URL", server.job_done_url.replace("$RUNPOD_POD_ID", "pod-1") + ) + + +@pytest.mark.asyncio +async def test_refresh_waits_for_an_earlier_slow_upload(monkeypatch): + """A normal job A whose result is still uploading must land before a later + refreshing job B sends stopPod: once the platform sees stopPod it can + restart the worker and kill A's upload.""" + server = PerJobUploadAiApi( + [generate_job("A", n=1), generate_job("B", n=2, refresh=True)], + slow_jobs={"A"}, + ) + await server.start() + _point_sdk_at(monkeypatch, server) + scaler = rp_scale.JobScaler({"handler": _sync_handler}) + run = asyncio.create_task(scaler.run()) + try: + _, pending = await asyncio.wait({run}, timeout=DEADLINE) + assert server.takes == ["A", "B"] + assert server.done_order == ["A", "B"], ( + "B's stopPod reached the platform before A's result landed" + ) + assert server.done["A"] == {"output": {"echo": 1}} + assert server.done["B"] == {"output": {"echo": 2}, "stopPod": True} + assert not pending + finally: + scaler.kill_worker() + run.cancel() + await asyncio.gather(run, return_exceptions=True) + await server.close() + + +@pytest.mark.asyncio +async def test_pending_uploads_are_bounded(monkeypatch): + """A stalled result endpoint must not let a single-slot worker accumulate + unbounded in-flight uploads; intake pauses at the ceiling.""" + monkeypatch.setattr(rp_scale, "MAX_PENDING_RESULT_UPLOADS", 2) + server = BlockingUploadAiApi([generate_job(f"J{i}", n=i) for i in range(10)]) + await server.start() + _point_sdk_at(monkeypatch, server) + scaler = rp_scale.JobScaler({"handler": _sync_handler}) + run = asyncio.create_task(scaler.run()) + try: + # Let uploads pile up against the ceiling. + for _ in range(100): + if server.uploading >= 2: + break + await asyncio.sleep(0.01) + await asyncio.sleep(0.1) # give intake every chance to overrun + assert server.max_uploading == 2, ( + f"uploads were not bounded: {server.max_uploading} in flight" + ) + assert server.done == {}, "no upload should have completed yet" + + server.release.set() # drain + await asyncio.wait_for(server.polled_empty.wait(), timeout=DEADLINE) + deadline = time.monotonic() + DEADLINE + while len(server.done) < 10 and time.monotonic() < deadline: + await asyncio.sleep(0.01) + assert len(server.done) == 10 + finally: + server.release.set() + scaler.kill_worker() + run.cancel() + await asyncio.gather(run, return_exceptions=True) + await server.close() + + +@pytest.mark.asyncio +async def test_set_scale_waits_for_pending_uploads(monkeypatch): + """A concurrency change must not swap jobs_queue while a result upload is + still in flight: the upload's task_done() would target the new queue.""" + monkeypatch.setattr(rp_scale, "JobsProgress", DummyProgress) + scaler = rp_scale.JobScaler({"handler": _sync_handler}) + original_queue = scaler.jobs_queue + scaler._sending_results.add("A") # a result still uploading + scaler.concurrency_modifier = lambda _current: 5 + + task = asyncio.create_task(scaler.set_scale()) + try: + await asyncio.sleep(0.05) + assert not task.done(), "set_scale resized while an upload was in flight" + assert scaler.jobs_queue is original_queue + + scaler._sending_results.discard("A") # upload finished + await asyncio.wait_for(task, timeout=DEADLINE) + assert scaler.jobs_queue is not original_queue + assert scaler.jobs_queue.maxsize == 5 + finally: + task.cancel() + await asyncio.gather(task, return_exceptions=True) + + @pytest.mark.asyncio async def test_custom_jobs_handler_keeps_the_slot_until_it_returns(): """Local-test jobs handlers have no on_result_ready; they keep the old contract.""" From 10eae578938784678b67176f67add96ad1f2dbe9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dean=20Qui=C3=B1anola?= Date: Mon, 16 Nov 2026 12:06:19 -0800 Subject: [PATCH 3/3] refactor(serverless): avoid ineffectual-statement flag on awaited callback Store the awaited on_result_ready result in an assignment instead of a bare await expression statement, so static analysis no longer reports it as having no effect. No behavior change. --- runpod/serverless/modules/rp_job.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/runpod/serverless/modules/rp_job.py b/runpod/serverless/modules/rp_job.py index 91241178c..083a9a6ff 100644 --- a/runpod/serverless/modules/rp_job.py +++ b/runpod/serverless/modules/rp_job.py @@ -252,9 +252,11 @@ async def handle_job( rp_debugger.clear_debugger_output() if on_result_ready is not None: - maybe_awaitable = on_result_ready(job_result) - if inspect.isawaitable(maybe_awaitable): - await maybe_awaitable + callback_result = on_result_ready(job_result) + if inspect.isawaitable(callback_result): + # Await the callback's coroutine for its side effects (freeing the + # slot, draining earlier uploads); it resolves to None. + callback_result = await callback_result # Send the job result back to JOB_DONE_URL await send_result(session, job_result, job, is_stream=is_stream)