Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions queue_job/README.rst
Original file line number Diff line number Diff line change
Expand Up @@ -213,6 +213,9 @@ The execution of channels by the job runner is defined by:
``--http-interface`` or ``localhost`` if unset
- ``ODOO_QUEUE_JOB_HTTP_AUTH_USER=jobrunner``, default empty
- ``ODOO_QUEUE_JOB_HTTP_AUTH_PASSWORD=s3cr3t``, default empty
- ``ODOO_QUEUE_JOB_METRICS_LISTEN_ADDRESS=127.0.0.1``, default
127.0.0.1
- ``ODOO_QUEUE_JOB_METRICS_PORT=9000``, default empty

- Using the Odoo configuration file (set either ``channels``, either
``max_capacity`` and/or ``db_max_capacity``)
Expand Down
2 changes: 1 addition & 1 deletion queue_job/__manifest__.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
"license": "LGPL-3",
"category": "Generic Modules",
"depends": ["mail", "base_sparse_field", "web"],
"external_dependencies": {"python": ["requests"]},
"external_dependencies": {"python": ["requests", "prometheus_client"]},
"data": [
"security/security.xml",
"security/ir.model.access.csv",
Expand Down
91 changes: 89 additions & 2 deletions queue_job/jobrunner/channels.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,8 @@
# Copyright 2015-2016 Camptocamp SA
# License LGPL-3.0 or later (http://www.gnu.org/licenses/lgpl.html)
import logging
import math
import weakref
from collections import namedtuple
from dataclasses import asdict, dataclass
from functools import total_ordering
Expand All @@ -10,6 +12,7 @@

from ..exception import ChannelNotFound
from ..job import CANCELLED, DONE, ENQUEUED, FAILED, PENDING, STARTED, WAIT_DEPENDENCIES
from . import metrics

RELOAD_PAYLOAD = "reload"
NOT_DONE = (WAIT_DEPENDENCIES, PENDING, ENQUEUED, STARTED, FAILED)
Expand Down Expand Up @@ -441,6 +444,7 @@ def __init__(
self._queue = ChannelQueue()
self._running = set()
self._failed = set()
self._waiting_dependencies = set()
self._pause_until = 0 # utc seconds since the epoch
self.capacity = (
capacity if (capacity is not None) else (parent and parent.capacity_default)
Expand All @@ -450,6 +454,57 @@ def __init__(
self.sequential = sequential or (parent and parent.sequential_default)
self.sequential_default = sequential_default
self.paused = paused
self._metrics_labels = None

def register_metrics(self, db_name) -> None:
"""Register gauges for a specific database in this channel."""
self.unregister_metrics()
self._metrics_labels = {
"db": metrics.db_label(db_name),
"channel": self.fullname,
}
metrics.channel_capacity.labels(**self._metrics_labels).set_function(
weakref.proxy(self)._capacity_gauge
)
metrics.channel_pending.labels(**self._metrics_labels).set_function(
weakref.proxy(self)._pending_gauge
)
metrics.channel_running.labels(**self._metrics_labels).set_function(
weakref.proxy(self)._running_gauge
)
metrics.channel_failed.labels(**self._metrics_labels).set_function(
weakref.proxy(self)._failed_gauge
)
metrics.channel_waiting_dependencies.labels(
**self._metrics_labels
).set_function(weakref.proxy(self)._waiting_dependencies_gauge)

def unregister_metrics(self) -> None:
if not self._metrics_labels:
return
metrics.channel_capacity.remove_by_labels(self._metrics_labels)
metrics.channel_pending.remove_by_labels(self._metrics_labels)
metrics.channel_running.remove_by_labels(self._metrics_labels)
metrics.channel_failed.remove_by_labels(self._metrics_labels)
metrics.channel_waiting_dependencies.remove_by_labels(self._metrics_labels)
self._metrics_labels = None

def _capacity_gauge(self) -> float:
if self.capacity is None:
return math.inf

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Note to self: test that prometheus client renders math.inf correctly

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It renders +Inf. Now need to see what prometheus and grafana do with that.

return self.capacity

def _pending_gauge(self) -> float:
return len(self._queue)

def _running_gauge(self) -> float:
return len(self._running)

def _failed_gauge(self) -> float:
return len(self._failed)

def _waiting_dependencies_gauge(self) -> float:
return len(self._waiting_dependencies)

@property
def sequential(self):
Expand Down Expand Up @@ -494,20 +549,22 @@ def get_subchannel_by_name(self, subchannel_name):

def __str__(self):
capacity = "∞" if self.capacity is None else str(self.capacity)
return "%s(C:%s,Q:%d,R:%d,F:%d%s)" % (
return "%s(C:%s,Q:%d,R:%d,F:%d%s),W:%s)" % (
self.fullname,
capacity,
len(self._queue),
len(self._running),
len(self._failed),
",paused" if self.paused else "",
len(self._waiting_dependencies),
)

def remove(self, job):
"""Remove a job from the channel."""
self._queue.remove(job)
self._running.discard(job)
self._failed.discard(job)
self._waiting_dependencies.discard(job)
if self.parent:
self.parent.remove(job)

Expand All @@ -529,6 +586,7 @@ def set_pending(self, job):
self._queue.add(job)
self._running.discard(job)
self._failed.discard(job)
self._waiting_dependencies.discard(job)
if self.parent:
self.parent.remove(job)
_logger.debug("job %s marked pending in channel %s", job.uuid, self)
Expand All @@ -542,6 +600,7 @@ def set_running(self, job):
self._queue.remove(job)
self._running.add(job)
self._failed.discard(job)
self._waiting_dependencies.discard(job)
if self.parent:
self.parent.set_running(job)
_logger.debug("job %s marked running in channel %s", job.uuid, self)
Expand All @@ -552,10 +611,23 @@ def set_failed(self, job):
self._queue.remove(job)
self._running.discard(job)
self._failed.add(job)
self._waiting_dependencies.discard(job)
if self.parent:
self.parent.remove(job)
_logger.debug("job %s marked failed in channel %s", job.uuid, self)

def set_waiting_dependencies(self, job):
if job not in self._waiting_dependencies:
self._queue.remove(job)
self._running.discard(job)
self._failed.discard(job)
self._waiting_dependencies.add(job)
if self.parent:
self.parent.remove(job)
_logger.debug(
"job %s marked waiting dependencies in channel %s", job.uuid, self
)

def has_capacity(self):
if self.paused:
return False
Expand Down Expand Up @@ -1179,7 +1251,7 @@ def notify(
job.channel.set_failed(job)
elif state == WAIT_DEPENDENCIES:
# wait until all parent jobs are done
pass
job.channel.set_waiting_dependencies(job)
else:
_logger.error("unexpected state %s for job %s", state, job)

Expand All @@ -1205,3 +1277,18 @@ def get_wakeup_time(self):
def running_count(self) -> int:
"""Number of jobs currently running"""
return len(self._root_channel._running)

def _jobs_to_do_gauge(self) -> float:
return len(self._jobs_by_uuid)

def register_metrics(self, db_name):
for channel in self._channels_by_name.values():
channel.register_metrics(db_name)
metrics.jobs_to_do.labels(db=metrics.db_label(db_name)).set_function(
self._jobs_to_do_gauge
)

def unregister_metrics(self, db_name):
for channel in self._channels_by_name.values():
channel.unregister_metrics()
metrics.jobs_to_do.remove_by_labels({"db": metrics.db_label(db_name)})
78 changes: 78 additions & 0 deletions queue_job/jobrunner/metrics.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,78 @@
import socket
from http.server import HTTPServer

from prometheus_client import CollectorRegistry, Counter, Gauge, MetricsHandler

_registry = CollectorRegistry()

_channel_label_names = ("db", "channel")
channel_capacity = Gauge(
"queue_job_channel_capacity",
documentation="Channel Capacity",
labelnames=_channel_label_names,
registry=_registry,
)
channel_pending = Gauge(
"queue_job_channel_pending",
documentation="Pending jobs in channel",
labelnames=_channel_label_names,
registry=_registry,
)
channel_running = Gauge(
"queue_job_channel_running",
documentation="Running jobs in channel",
labelnames=_channel_label_names,
registry=_registry,
)
channel_failed = Gauge(
"queue_job_channel_failed",
documentation="Failed jobs in channel",
labelnames=_channel_label_names,
registry=_registry,
)
channel_waiting_dependencies = Gauge(
"queue_job_channel_waiting_dependencies",
documentation="Jobs waiting for dependencies in channel",
labelnames=_channel_label_names,
registry=_registry,
)

jobs_to_do = Gauge(
"queue_job_jobs_to_do",
documentation=(
"Number of jobs waiting to be done (including running and failed jobs)"
),
labelnames=("db",),
registry=_registry,
)
jobs_scheduled_total = Counter(
"queue_job_jobs_scheduled_total",
documentation=(
"Total number of jobs scheduled for execution (asked Odoo to run job)"
),
labelnames=("db",),
registry=_registry,
)
dead_jobs_requeued_total = Counter(
"queue_job_dead_jobs_requeued_total",
documentation=("Total number of dead jobs requeued"),
labelnames=("db",),
registry=_registry,
)


def db_label(db_name: str | None) -> str:
return db_name or ""


def make_metrics_server(bind_addr, port) -> HTTPServer:
infos = socket.getaddrinfo(
bind_addr,
port,
type=socket.SOCK_STREAM,
flags=socket.AI_PASSIVE,
)
_, _, _, _, sockaddr = next(iter(infos))
server = HTTPServer(sockaddr, MetricsHandler.factory(_registry))
server.socket.setblocking(False)
return server
Loading
Loading