Repository navigation
feat(orchestrator): Refresh a container execution as soon as its pod or job changes #428
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: master
Are you sure you want to change the base?
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -4,6 +4,7 @@ | |
| import logging | ||
| import itertools | ||
| import os | ||
| import threading | ||
| import time | ||
| import traceback | ||
| import typing | ||
|
|
@@ -36,6 +37,10 @@ | |
| DYNAMIC_DATA_SECRET_KEY = "secret" | ||
| DYNAMIC_DATA_SECRET_NAME_KEY = "name" | ||
|
|
||
| _CHANGED_CONTAINER_EXECUTION_PRIORITY = datetime.timedelta(minutes=2) | ||
|
|
||
| _MAX_CHANGED_CONTAINER_EXECUTION_IDS = 10_000 | ||
|
|
||
|
|
||
| class OrchestratorError(RuntimeError): | ||
| pass | ||
|
|
@@ -94,6 +99,11 @@ def __init__( | |
| self._max_queue_batch_size = _max_queue_batch_size | ||
| self._max_queue_batch_duration = _max_queue_batch_duration | ||
| self._queued_execution_interceptor = queued_execution_interceptor | ||
| self._changed_container_execution_ids: set[str] = set() | ||
| self._changed_container_execution_ids_lock = threading.Lock() | ||
| self._launcher.subscribe_to_container_execution_changes( | ||
| listener=self._record_container_execution_change | ||
| ) | ||
|
|
||
| def run_loop(self): | ||
| while True: | ||
|
|
@@ -197,6 +207,7 @@ def internal_process_queued_executions_queue(self, session: orm.Session): | |
| return True | ||
|
|
||
| def internal_process_running_executions_queue(self, session: orm.Session): | ||
| self._move_changed_container_executions_to_front(session=session) | ||
| _logger.debug("Selecting running container execution to process.)") | ||
|
|
||
| # Select only the ID to avoid loading large JSON columns into the sort buffer. | ||
|
|
@@ -341,6 +352,51 @@ def internal_process_running_executions_queue(self, session: orm.Session): | |
| ) | ||
| return True | ||
|
|
||
| def _record_container_execution_change( | ||
| self, change: launcher_interfaces.ContainerExecutionChange | ||
| ) -> None: | ||
| """Records a container execution the launcher saw change, for the next running sweep to check first. | ||
|
|
||
| Runs on the launcher's thread; past the cap new ids are dropped, so those executions just wait their normal turn. | ||
| """ | ||
| with self._changed_container_execution_ids_lock: | ||
| if ( | ||
| len(self._changed_container_execution_ids) | ||
| < _MAX_CHANGED_CONTAINER_EXECUTION_IDS | ||
| ): | ||
| self._changed_container_execution_ids.add(change.container_execution_id) | ||
|
|
||
| def _move_changed_container_executions_to_front(self, session: orm.Session) -> None: | ||
| """Moves each pending or running execution the launcher saw change to `_CHANGED_CONTAINER_EXECUTION_PRIORITY` ago, unless it's already further ahead. | ||
|
|
||
| Changes are served in the order recorded, an execution that keeps changing can't starve the rest, and a change | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
Clever idea. A bit non-obvious to prove, but I've verified it's correct. |
||
| seen mid-refresh (such as the container finishing) moves that execution forward again. | ||
| """ | ||
| with self._changed_container_execution_ids_lock: | ||
| container_execution_ids = self._changed_container_execution_ids | ||
| self._changed_container_execution_ids = set() | ||
| if not container_execution_ids: | ||
| return | ||
| front_of_queue_last_processed_at = ( | ||
| _get_current_time() - _CHANGED_CONTAINER_EXECUTION_PRIORITY | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
Unlikely to be a big problem. |
||
| ) | ||
| session.execute( | ||
| sql.update(bts.ContainerExecution) | ||
| .where( | ||
| bts.ContainerExecution.id.in_(container_execution_ids), | ||
| bts.ContainerExecution.status.in_( | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Clever. This fixes a small race condition (caused by multi-threaded informers). Maybe we can leave a short comment about this. The race condition is as follows:
The case is really minor (incorrect last_processed_at is not a big deal), but I'm glad you avoided this issue.
Collaborator
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. 🤖 AI-assisted: Kept in c7b30da, and the case is now in the |
||
| ( | ||
| bts.ContainerExecutionStatus.PENDING, | ||
| bts.ContainerExecutionStatus.RUNNING, | ||
| ) | ||
| ), | ||
| bts.ContainerExecution.last_processed_at | ||
| > front_of_queue_last_processed_at, | ||
| ) | ||
| .values(last_processed_at=front_of_queue_last_processed_at) | ||
| ) | ||
| session.commit() | ||
|
|
||
| def internal_process_one_queued_execution( | ||
| self, session: orm.Session, execution: bts.ExecutionNode | ||
| ): | ||
|
|
||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
This might behave differently from expectations in certain cases. For example if the queue is long and a normal full running execution queue sweep takes more than
_CHANGED_CONTAINER_EXECUTION_PRIORITY, then the recently changed executions will be processed with arbitrary delay.This seems to be the natural conflict between processing the "normal" queue and the listener-based queue. You can make the problem fixable by making
_CHANGED_CONTAINER_EXECUTION_PRIORITYconfigurable.Non-blocking.