Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 23 additions & 2 deletions runpod/serverless/modules/rp_job.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
import json
import os
import traceback
from typing import Any, AsyncGenerator, Callable, Dict, Optional, Union, List
from typing import Any, AsyncGenerator, Awaitable, Callable, Dict, Optional, Union, List

import aiohttp

Expand Down Expand Up @@ -194,7 +194,21 @@
return jobs


async def handle_job(session: ClientSession, config: Dict[str, Any], job) -> dict:
async def handle_job(
session: ClientSession,
config: Dict[str, Any],
job,
on_result_ready: Optional[
Callable[[Dict[str, Any]], Optional[Awaitable[None]]]
] = None,
) -> dict:
"""Run a job and send its result.

on_result_ready, if given, is called with the final result just before it
is sent, so the caller can free the job's slot and take the next job
while the result upload is still in flight. It may return an awaitable,
which is awaited before the result is sent.
"""
if is_generator(config["handler"]):
is_stream = True
generator_output = run_job_generator(config["handler"], job)
Expand Down Expand Up @@ -237,6 +251,13 @@
log.debug("rp_debugger | Flag not set, skipping debugger output.", job["id"])
rp_debugger.clear_debugger_output()

if on_result_ready is not None:
callback_result = on_result_ready(job_result)
if inspect.isawaitable(callback_result):
# Await the callback's coroutine for its side effects (freeing the
# slot, draining earlier uploads); it resolves to None.
callback_result = await callback_result
Comment thread
deanq marked this conversation as resolved.
Resolved

# Send the job result back to JOB_DONE_URL
await send_result(session, job_result, job, is_stream=is_stream)
return job_result
Expand Down
96 changes: 82 additions & 14 deletions runpod/serverless/modules/rp_scale.py
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,13 @@
# wait so one long job can't hold the other slots idle, and so set_scale still
# runs as often as it did under the old fixed sleep.
FULL_WAIT_SECONDS = 1
# Cap the result uploads that may overlap new work. The shared HTTP client sets
# no connection limit (TCPConnector(limit=0)) and a 600 s timeout, so without a
# ceiling a stalled result endpoint would let a worker retain unbounded results
# and open unbounded connections while it keeps taking jobs. Normal pipelines
# stay well under this; it only applies backpressure to a persistently slow
# result endpoint.
MAX_PENDING_RESULT_UPLOADS = 50


def _handle_uncaught_exception(exc_type, exc_value, exc_traceback):
Expand Down Expand Up @@ -80,6 +87,9 @@ def __init__(self, config: dict[str, Any]):
# maps in-progress job ids to their running tasks so individual jobs
# can be stopped without killing the whole worker
self.jobs_tasks: dict[str, asyncio.Task[Any]] = {}
# Jobs whose handler finished and whose result is still being sent.
# They no longer hold a slot, so the next job-take overlaps the upload.
self._sending_results: set[str] = set()

self.stop_signals_fetcher = get_stop_signals
self.stop_signals_fetcher_timeout = 90
Expand Down Expand Up @@ -124,8 +134,9 @@ async def set_scale(self):
# no need to resize
return

while self.current_occupancy() > 0:
# not safe to scale when jobs are in flight
while self.current_occupancy() > 0 or self._sending_results:
# not safe to scale while jobs are in flight or results are still
# uploading: a swapped queue would receive the upload's task_done()
await asyncio.sleep(1)
continue

Expand Down Expand Up @@ -416,8 +427,15 @@ async def run_jobs(self, session: ClientSession):
# Clear before draining: a job queued after this point re-arms it.
self._jobs_available.clear()

# Fetch as many jobs as the concurrency allows
while len(tasks) < self.current_concurrency and not self.jobs_queue.empty():
# Fetch as many jobs as the concurrency allows; a task that is only
# sending its result no longer counts against it. Stop once too many
# uploads are already in flight so a slow result endpoint cannot
# accumulate unbounded tasks, connections, and retained results.
while (
len(tasks) - len(self._sending_results) < self.current_concurrency
Comment thread
deanq marked this conversation as resolved.
Resolved
and len(self._sending_results) < MAX_PENDING_RESULT_UPLOADS
and not self.jobs_queue.empty()
):
job = await self.jobs_queue.get()
# Create a new task for each job and track it by job id
task = asyncio.create_task(self.handle_job(session, job))
Expand Down Expand Up @@ -562,15 +580,16 @@ async def handle_job(self, session: ClientSession, job: dict[str, Any]):
)
return

job_result = await self.jobs_handler(session, self.config, job)

# Stop intake before the finally frees this slot: the platform
# restarts the worker on stopPod, killing any job taken meanwhile.
if self.config.get("refresh_worker", False) or (
isinstance(job_result, dict) and job_result.get("stopPod")
):
log.info("Refresh requested, taking no more jobs.", job["id"])
self.kill_worker()
if self.jobs_handler is handle_job:
job_result = await self.jobs_handler(
session,
self.config,
job,
on_result_ready=lambda result: self._release_slot(job, result),
)
else:
job_result = await self.jobs_handler(session, self.config, job)
self._stop_intake_if_refreshing(job, job_result)

except asyncio.CancelledError:
log.info("Job stopped.", job["id"])
Expand All @@ -585,13 +604,62 @@ async def handle_job(self, session: ClientSession, job: dict[str, Any]):
self.jobs_queue.task_done()

# Job is no longer in progress
self.job_progress.remove(job)
if job["id"] in self._sending_results:
self._sending_results.discard(job["id"])
else:
self.job_progress.remove(job)
self.jobs_tasks.pop(job["id"], None)
self._slot_freed.set()

log.debug("Finished Job", job["id"])
_reset_batch_id(batch_id_token)

def _requests_refresh(self, job_result: Any) -> bool:
"""Whether this job asks the worker to refresh (restart) after it."""
return bool(
self.config.get("refresh_worker", False)
or (isinstance(job_result, dict) and job_result.get("stopPod"))
)

def _stop_intake_if_refreshing(self, job: dict[str, Any], job_result: Any) -> None:
"""Stop intake before this job's slot is freed: the platform restarts
the worker on stopPod, killing any job taken meanwhile."""
if not self.is_alive():
return
if self._requests_refresh(job_result):
log.info("Refresh requested, taking no more jobs.", job["id"])
self.kill_worker()

async def _release_slot(self, job: dict[str, Any], job_result: Any) -> None:
"""Free a finished job's slot before its result is sent.

The result upload takes a full round trip (~0.3 s measured on
serverless), and a single-slot worker used to wait it out before
asking for the next job. Taking the next job while the upload is in
flight hides it for every job queued behind another.
"""
refreshing = self._requests_refresh(job_result)
self._stop_intake_if_refreshing(job, job_result)
Comment thread
deanq marked this conversation as resolved.
Resolved
self._sending_results.add(job["id"])
self.job_progress.remove(job)
Comment thread
deanq marked this conversation as resolved.
Resolved
self._slot_freed.set()
if refreshing:
# Let earlier uploads land before this refresh result reaches the
# platform: once it sees stopPod it can restart the worker and kill
# a result still in flight.
await self._drain_earlier_uploads(job["id"])

async def _drain_earlier_uploads(self, job_id: str) -> None:
"""Wait for every in-flight result upload other than this job's."""
pending = [
task
for other_id in list(self._sending_results)
if other_id != job_id
and (task := self.jobs_tasks.get(other_id)) is not None
]
if pending:
await asyncio.gather(*pending, return_exceptions=True)

async def _wait_for_prestart(self) -> None:
"""Wait for prestart to finish or shutdown to begin."""
ready = asyncio.create_task(self._prestart_ready.wait())
Expand Down
Loading
Loading