diff --git a/doc/lowlevel-guide.md b/doc/lowlevel-guide.md index 64101ca5..1604d07a 100644 --- a/doc/lowlevel-guide.md +++ b/doc/lowlevel-guide.md @@ -28,6 +28,7 @@ 10. [没做的(如实列出)](#10-没做的如实列出) 11. [定时任务(cron)](#11-定时任务cron) 12. [进程监管(process supervisor)](#12-进程监管process-supervisor) +13. [作业队列(job queue)](#13-作业队列job-queue) --- @@ -429,6 +430,42 @@ sup.spawn(spec, on_output, on_exit); --- +## 13. 作业队列(job queue) + +`uvcpp_job_queue`(`#include `)是"持久化长任务"的第一批: + +```cpp +// doc-snippet: fragment — 装配摘录,不是完整翻译单元。 +uvcpp::uvcpp_job_queue queue(&loop); // 默认内存 store +queue.register_handler("media.transcode", [](uvcpp::uvcpp_job_context& ctx) { + // 在 libuv 线程池上同步执行;长循环里轮询 ctx.cancelled() +}); +queue.start(); // 恢复上次未完成的记录 + +uvcpp::uvcpp_job_submit req; +req.type = "media.transcode"; +req.concurrency_key = "resource-42"; // 同一 key 严格串行 +req.payload = "{}"; +req.max_retry = 3; +req.backoff_ms = 1000; // 指数退避,封顶 1 小时 +req.timeout_ms = 10 * 60 * 1000; +queue.submit(req); +``` + +- 状态:PENDING / RUNNING / SUCCESS / FAILED / CANCELLED;每次变更写 store 并回调 + `on_state()`。 +- handler 在 **worker 线程**上同步跑;超时/取消是**协作式**的(置 `ctx.cancelled()`, + 队列不杀线程)。 +- `concurrency_key` 非空时同一 key 同时只跑一个;`max_concurrent` 默认 4。 +- `max_queue > 0` 时 PENDING 达上限 `submit()` 拒绝(`stats().rejected`)。 +- `start()` 从 store 读回记录;上次停在 RUNNING 的恢复成 PENDING 并标 + `recovered after restart`。 +- store 接口 `uvcpp_job_store` 可替换(本文件带内存实现);SQLite/MySQL 适配器与 + 异步 handler 是 #67 的后续。 +- **析构前**必须 `stop()` 且 `running == 0`。用例:`job_queue_func.cpp`。 + +--- + 相关文档:[net 网络层指南](./net-guide.md)、[TLS 与证书指南](./ssl-guide.md)、 [项目 README](../README.zh.md)。 diff --git a/src/uvcpp/uvcpp_job_queue.cpp b/src/uvcpp/uvcpp_job_queue.cpp new file mode 100644 index 00000000..d2530d91 --- /dev/null +++ b/src/uvcpp/uvcpp_job_queue.cpp @@ -0,0 +1,536 @@ +/** + * @file src/uvcpp/uvcpp_job_queue.cpp + * @brief 作业队列的实现(内存记录 + uv 线程池执行)。 + * @author zhuweiye + * @version 1.0.0 + */ + +#include "uvcpp/uvcpp_job_queue.h" + +#include +#include + +#include + +#include +#include +#include +#include +#include +#include +#include + +namespace uvcpp { + +uvcpp_job_record::uvcpp_job_record() + : id(0), + state(static_cast(uvcpp_job_state::PENDING)), + attempt(0), + max_retry(0), + timeout_ms(0), + backoff_ms(1000), + not_before_ms(0) {} + +uvcpp_job_submit::uvcpp_job_submit() + : max_retry(0), timeout_ms(0), backoff_ms(1000) {} + +uvcpp_job_stats::uvcpp_job_stats() + : pending(0), + running(0), + success(0), + failed(0), + cancelled(0), + retried(0), + rejected(0) {} + +// =========================================================================== +// 内存 store +// =========================================================================== + +struct uvcpp_memory_job_store::impl { + mutable std::mutex mu; + std::map rows; +}; + +uvcpp_memory_job_store::uvcpp_memory_job_store() : impl_(new impl()) {} +uvcpp_memory_job_store::~uvcpp_memory_job_store() {} + +bool uvcpp_memory_job_store::load_all(std::vector& out) { + std::lock_guard lk(impl_->mu); + out.clear(); + out.reserve(impl_->rows.size()); + for (std::map::const_iterator it = + impl_->rows.begin(); + it != impl_->rows.end(); ++it) { + out.push_back(it->second); + } + return true; +} + +bool uvcpp_memory_job_store::save(const uvcpp_job_record& rec) { + std::lock_guard lk(impl_->mu); + impl_->rows[rec.id] = rec; + return true; +} + +bool uvcpp_memory_job_store::erase(uint64_t id) { + std::lock_guard lk(impl_->mu); + impl_->rows.erase(id); + return true; +} + +// =========================================================================== +// 上下文 +// =========================================================================== + +namespace { + +struct job_impl { + uvcpp_job_record rec; + std::atomic cancel_requested; + std::atomic timed_out; + std::string handler_error; ///< 仅 worker 写、after_work 读 + bool has_handler_error; + uvcpp_work* work; + uv_timer_t* timeout_timer; + + job_impl() : cancel_requested(false), timed_out(false), + has_handler_error(false), work(nullptr), + timeout_timer(nullptr) {} +}; + +} // namespace + +struct uvcpp_job_context::impl { + job_impl* j; +}; + +namespace { + +} // namespace + +uvcpp_job_context::uvcpp_job_context() : impl_(new impl()) { + impl_->j = nullptr; +} +uvcpp_job_context::~uvcpp_job_context() { delete impl_; } + +bool uvcpp_job_context::cancelled() const { + return impl_ != nullptr && impl_->j != nullptr && + impl_->j->cancel_requested.load(); +} + +const uvcpp_job_record& uvcpp_job_context::job() const { + static const uvcpp_job_record empty; + return impl_ != nullptr && impl_->j != nullptr ? impl_->j->rec : empty; +} + +// =========================================================================== +// 队列 +// =========================================================================== + +struct uvcpp_job_queue::impl { + explicit impl(uvcpp_loop* l, std::shared_ptr st) + : loop(l), + store(std::move(st)), + max_concurrent(4), + max_queue(0), + running(0), + next_id(1), + started(false), + backoff_timer(nullptr), + backoff_init(false), + retried(0), + rejected(0) {} + + uvcpp_loop* loop; + std::shared_ptr store; + std::map > jobs; + std::map handlers; + std::set running_keys; + size_t max_concurrent; + size_t max_queue; + size_t running; + uint64_t next_id; + bool started; + uv_timer_t* backoff_timer; + bool backoff_init; + uint64_t retried; + uint64_t rejected; + uvcpp_job_state_cb state_cb; + + static int64_t now_ms() { + return static_cast( + std::chrono::duration_cast( + std::chrono::system_clock::now().time_since_epoch()) + .count()); + } + + void notify(const uvcpp_job_record& rec) { + if (state_cb) state_cb(rec); + } + + void persist(job_impl* j) { + if (store) store->save(j->rec); + } + + static void on_backoff(uv_timer_t* t) { + impl* self = static_cast(t->data); + if (self != nullptr) self->dispatch(); + } + + void schedule_backoff() { + if (!backoff_init) return; + int64_t earliest = 0; + for (std::map >::iterator it = + jobs.begin(); + it != jobs.end(); ++it) { + if (it->second->rec.state != + static_cast(uvcpp_job_state::PENDING)) + continue; + if (it->second->rec.not_before_ms <= 0) continue; + if (earliest == 0 || it->second->rec.not_before_ms < earliest) + earliest = it->second->rec.not_before_ms; + } + if (earliest == 0) return; + int64_t delay = earliest - now_ms(); + if (delay < 1) delay = 1; + uv_timer_start(backoff_timer, on_backoff, static_cast(delay), 0); + } + + void dispatch(); + void run_job(const std::shared_ptr& j); + void finish_job(const std::shared_ptr& j, bool ok); + void release_running(job_impl* j); +}; + +void uvcpp_job_queue::impl::release_running(job_impl* j) { + if (running > 0) --running; + if (!j->rec.concurrency_key.empty()) { + running_keys.erase(j->rec.concurrency_key); + } +} + +void uvcpp_job_queue::impl::dispatch() { + if (!started) return; + bool progressed = true; + while (progressed && running < max_concurrent) { + progressed = false; + const int64_t now = now_ms(); + for (std::map >::iterator it = + jobs.begin(); + it != jobs.end(); ++it) { + job_impl* j = it->second.get(); + if (j->rec.state != static_cast(uvcpp_job_state::PENDING)) continue; + if (j->rec.not_before_ms > 0 && j->rec.not_before_ms > now) continue; + if (!j->rec.concurrency_key.empty() && + running_keys.find(j->rec.concurrency_key) != running_keys.end()) { + continue; + } + run_job(it->second); + progressed = true; + break; + } + } + schedule_backoff(); +} + +void uvcpp_job_queue::impl::run_job(const std::shared_ptr& sp) { + job_impl* j = sp.get(); + j->rec.state = static_cast(uvcpp_job_state::RUNNING); + j->rec.attempt += 1; + j->rec.error.clear(); + j->has_handler_error = false; + j->cancel_requested.store(false); + j->timed_out.store(false); + persist(j); + notify(j->rec); + ++running; + if (!j->rec.concurrency_key.empty()) { + running_keys.insert(j->rec.concurrency_key); + } + + if (j->rec.timeout_ms > 0) { + j->timeout_timer = new uv_timer_t(); + uv_timer_init(reinterpret_cast(loop->get_handle()), + j->timeout_timer); + j->timeout_timer->data = j; + uv_timer_start( + j->timeout_timer, + [](uv_timer_t* t) { + job_impl* jj = static_cast(t->data); + if (jj == nullptr) return; + jj->timed_out.store(true); + jj->cancel_requested.store(true); + }, + j->rec.timeout_ms, 0); + } + + uvcpp_job_handler handler; + std::map::iterator hit = + handlers.find(j->rec.type); + if (hit != handlers.end()) handler = hit->second; + + uvcpp_work* work = new uvcpp_work(); + j->work = work; + impl* self = this; + const int rc = work->queue_work( + loop, + [sp, handler](uvcpp_work*) { + job_impl* jj = sp.get(); + if (!handler) { + jj->has_handler_error = true; + jj->handler_error = "没有注册该类型的 handler"; + return; + } + uvcpp_job_context ctx; + ctx.impl_->j = jj; + try { + handler(ctx); + } catch (const std::exception& e) { + jj->has_handler_error = true; + jj->handler_error = e.what(); + } catch (...) { + jj->has_handler_error = true; + jj->handler_error = "handler 抛出未知异常"; + } + ctx.impl_->j = nullptr; + }, + [sp, self](uvcpp_work* w, int status) { + job_impl* jj = sp.get(); + delete w; + if (jj->timeout_timer != nullptr) { + uv_timer_stop(jj->timeout_timer); + uv_close(reinterpret_cast(jj->timeout_timer), + [](uv_handle_t* h) { + delete reinterpret_cast(h); + }); + jj->timeout_timer = nullptr; + } + jj->work = nullptr; + self->release_running(jj); + const bool ok = status == 0 && !jj->has_handler_error && + !jj->timed_out.load() && + !jj->cancel_requested.load(); + if (!ok && jj->has_handler_error) jj->rec.error = jj->handler_error; + self->finish_job(sp, ok); + self->dispatch(); + }); + if (rc != 0) { + delete work; + j->work = nullptr; + if (j->timeout_timer != nullptr) { + uv_timer_stop(j->timeout_timer); + uv_close(reinterpret_cast(j->timeout_timer), + [](uv_handle_t* h) { + delete reinterpret_cast(h); + }); + j->timeout_timer = nullptr; + } + release_running(j); + j->rec.error = "投递到线程池失败"; + finish_job(sp, false); + } +} + +void uvcpp_job_queue::impl::finish_job(const std::shared_ptr& sp, + bool ok) { + job_impl* j = sp.get(); + if (ok) { + j->rec.state = static_cast(uvcpp_job_state::SUCCESS); + } else if (j->cancel_requested.load() && !j->timed_out.load()) { + j->rec.state = static_cast(uvcpp_job_state::CANCELLED); + } else { + const bool retryable = j->rec.attempt <= j->rec.max_retry; + if (retryable) { + // 指数退避,封顶 1 小时。 + uint64_t backoff = j->rec.backoff_ms; + for (int k = 1; k < j->rec.attempt && backoff < 3600ull * 1000; ++k) { + backoff *= 2; + } + if (backoff > 3600ull * 1000) backoff = 3600ull * 1000; + j->rec.state = static_cast(uvcpp_job_state::PENDING); + j->rec.not_before_ms = now_ms() + static_cast(backoff); + if (j->rec.error.empty()) { + j->rec.error = j->timed_out.load() ? "timeout" : "failed"; + } + ++retried; + } else { + j->rec.state = static_cast(uvcpp_job_state::FAILED); + if (j->rec.error.empty()) { + j->rec.error = j->timed_out.load() ? "timeout" : "failed"; + } + } + } + persist(j); + notify(j->rec); +} + +uvcpp_job_queue::uvcpp_job_queue(uvcpp_loop* loop, + std::shared_ptr store) + : impl_(new impl(loop, store ? store : std::make_shared())) { + if (impl_->loop != nullptr) { + impl_->backoff_timer = new uv_timer_t(); + uv_timer_init(reinterpret_cast(impl_->loop->get_handle()), + impl_->backoff_timer); + impl_->backoff_timer->data = impl_; + impl_->backoff_init = true; + } +} + +uvcpp_job_queue::~uvcpp_job_queue() { + // 契约:析构前任务的 running 必须已经归零(work 回调持有 impl_)。 + if (impl_->backoff_init) { + uv_timer_stop(impl_->backoff_timer); + uv_close(reinterpret_cast(impl_->backoff_timer), + [](uv_handle_t* h) { delete reinterpret_cast(h); }); + impl_->backoff_timer = nullptr; + } + delete impl_; +} + +void uvcpp_job_queue::register_handler(const std::string& type, + const uvcpp_job_handler& handler) { + impl_->handlers[type] = handler; +} + +void uvcpp_job_queue::set_max_concurrent(size_t n) { + impl_->max_concurrent = n == 0 ? 1 : n; +} + +void uvcpp_job_queue::set_max_queue(size_t n) { impl_->max_queue = n; } + +uint64_t uvcpp_job_queue::submit(const uvcpp_job_submit& req, + std::string* err) { + if (impl_->loop == nullptr || req.type.empty()) { + if (err) *err = "没有循环或 type 为空"; + return 0; + } + size_t pending = 0; + for (std::map >::iterator it = + impl_->jobs.begin(); + it != impl_->jobs.end(); ++it) { + if (it->second->rec.state == static_cast(uvcpp_job_state::PENDING)) + ++pending; + } + if (impl_->max_queue > 0 && pending >= impl_->max_queue) { + ++impl_->rejected; + if (err) *err = "队列已满"; + return 0; + } + + std::shared_ptr j(new job_impl()); + j->rec.id = impl_->next_id++; + j->rec.type = req.type; + j->rec.payload = req.payload; + j->rec.concurrency_key = req.concurrency_key; + j->rec.state = static_cast(uvcpp_job_state::PENDING); + j->rec.attempt = 0; + j->rec.max_retry = req.max_retry < 0 ? 0 : req.max_retry; + j->rec.timeout_ms = req.timeout_ms; + j->rec.backoff_ms = req.backoff_ms; + j->rec.not_before_ms = 0; + impl_->jobs[j->rec.id] = j; + if (impl_->store) impl_->store->save(j->rec); + impl_->notify(j->rec); + impl_->dispatch(); + return j->rec.id; +} + +int uvcpp_job_queue::cancel(uint64_t id) { + std::map >::iterator it = + impl_->jobs.find(id); + if (it == impl_->jobs.end()) return -1; + job_impl* j = it->second.get(); + if (j->rec.state == static_cast(uvcpp_job_state::PENDING)) { + j->rec.state = static_cast(uvcpp_job_state::CANCELLED); + impl_->persist(j); + impl_->notify(j->rec); + impl_->dispatch(); + return 0; + } + if (j->rec.state == static_cast(uvcpp_job_state::RUNNING)) { + // 协作式:只能让 handler 自己尽早退出;状态在它返回后变 CANCELLED。 + j->cancel_requested.store(true); + return 0; + } + return -1; +} + +void uvcpp_job_queue::start() { + if (impl_->started) return; + impl_->started = true; + if (impl_->store) { + std::vector rows; + impl_->store->load_all(rows); + for (size_t i = 0; i < rows.size(); ++i) { + uvcpp_job_record rec = rows[i]; + if (rec.id == 0 || impl_->jobs.find(rec.id) != impl_->jobs.end()) { + if (rec.id >= impl_->next_id) impl_->next_id = rec.id + 1; + continue; + } + if (rec.state == static_cast(uvcpp_job_state::RUNNING)) { + // 上次进程在执行中退出:回到 PENDING,由业务决定重试/放弃。 + rec.state = static_cast(uvcpp_job_state::PENDING); + rec.error = "recovered after restart"; + } + std::shared_ptr j(new job_impl()); + j->rec = rec; + impl_->jobs[rec.id] = j; + if (rec.id >= impl_->next_id) impl_->next_id = rec.id + 1; + impl_->store->save(j->rec); + } + } + impl_->dispatch(); +} + +void uvcpp_job_queue::stop() { impl_->started = false; } + +void uvcpp_job_queue::on_state(const uvcpp_job_state_cb& cb) { + impl_->state_cb = cb; +} + +const uvcpp_job_record* uvcpp_job_queue::find(uint64_t id) const { + std::map >::const_iterator it = + impl_->jobs.find(id); + return it == impl_->jobs.end() ? nullptr : &it->second->rec; +} + +size_t uvcpp_job_queue::clear_finished() { + size_t n = 0; + for (std::map >::iterator it = + impl_->jobs.begin(); + it != impl_->jobs.end();) { + const int st = it->second->rec.state; + if (st == static_cast(uvcpp_job_state::SUCCESS) || + st == static_cast(uvcpp_job_state::FAILED) || + st == static_cast(uvcpp_job_state::CANCELLED)) { + if (impl_->store) impl_->store->erase(it->first); + it = impl_->jobs.erase(it); + ++n; + } else { + ++it; + } + } + return n; +} + +uvcpp_job_stats uvcpp_job_queue::stats() const { + uvcpp_job_stats s; + for (std::map >::const_iterator it = + impl_->jobs.begin(); + it != impl_->jobs.end(); ++it) { + switch (it->second->rec.state) { + case static_cast(uvcpp_job_state::PENDING): ++s.pending; break; + case static_cast(uvcpp_job_state::RUNNING): ++s.running; break; + case static_cast(uvcpp_job_state::SUCCESS): ++s.success; break; + case static_cast(uvcpp_job_state::FAILED): ++s.failed; break; + case static_cast(uvcpp_job_state::CANCELLED): ++s.cancelled; break; + default: break; + } + } + s.retried = impl_->retried; + s.rejected = impl_->rejected; + return s; +} + +} // namespace uvcpp diff --git a/src/uvcpp/uvcpp_job_queue.h b/src/uvcpp/uvcpp_job_queue.h new file mode 100644 index 00000000..14de16ab --- /dev/null +++ b/src/uvcpp/uvcpp_job_queue.h @@ -0,0 +1,208 @@ +/** + * @file src/uvcpp/uvcpp_job_queue.h + * @brief 持久化作业队列:同步 handler、并发键串行、重试退避、超时、取消(issue #67 第一批)。 + * @author zhuweiye + * @version 1.0.0 + * + * ```cpp + * auto store = std::make_shared(); + * uvcpp::uvcpp_job_queue queue(&loop, store); + * queue.register_handler("media.transcode", + * [](uvcpp::uvcpp_job_context& ctx) { /* 在 worker 上跑 *\/ }); + * queue.start(); // 恢复上次未完成的任务 + * + * uvcpp::uvcpp_job_submit req; + * req.type = "media.transcode"; + * req.concurrency_key = "resource-42"; // 同一 key 严格串行 + * req.payload = "{}"; + * req.timeout_ms = 10 * 60 * 1000; + * req.max_retry = 3; + * queue.submit(req, &err); + * ``` + * + * ## 语义 + * + * - **状态**:PENDING / RUNNING / SUCCESS / FAILED / CANCELLED;每次变更写 store + * 并回调 `on_state()`(loop 线程)。 + * - **handler 在 libuv 线程池里同步执行**(`uvcpp_work`);不要在里面阻塞循环 + * 线程的东西。异步 handler 是后续(#67 剩余)。 + * - **`concurrency_key`**:同一个非空 key 同时只允许一个 RUNNING;空 = 不限制。 + * 排队顺序按提交 id。 + * - **重试**:失败后 `attempt <= max_retry` 时回到 PENDING,`not_before = + * now + backoff_ms * 2^(attempt-1)`(上限 1 小时)。 + * - **超时 / 取消是协作式的**:到点/取消会把 `ctx.cancelled()` 置真;handler + * 应在长循环里轮询它并尽早返回。**队列不会杀线程**(也杀不了)。 + * - **重启恢复**:`start()` 从 store 读回记录;上次停在 RUNNING 的按 PENDING + * 处理(`error` 写一句恢复说明),由业务选择是否重试 —— 这是 #67 要求的 + * "进程崩溃后的恢复语义"的最小明确版本。 + * - **队列满**:`max_queue > 0` 且 PENDING 数达到它时 `submit()` 返回 0 并写 + * `err`(拒绝,不排队)。`rejected` 统计。 + * - **只在 loop 线程上调用**;handler 在 worker 线程,只能通过 ctx 读 + * `cancelled()` 与作业快照。 + */ + +#pragma once +#ifndef SRC_UVCPP_UVCPP_JOB_QUEUE_H +#define SRC_UVCPP_UVCPP_JOB_QUEUE_H + +#include +#include + +#include +#include +#include +#include +#include +#include + +namespace uvcpp { + +class uvcpp_loop; + +enum class uvcpp_job_state : int { + PENDING = 0, + RUNNING = 1, + SUCCESS = 2, + FAILED = 3, + CANCELLED = 4, +}; + +/// 一条作业的完整记录(store 持久化的就是它)。 +struct UVCPP_API uvcpp_job_record { + uvcpp_job_record(); + + uint64_t id; + std::string type; + std::string payload; + std::string concurrency_key; + std::string error; + int state; ///< `uvcpp_job_state` + int attempt; ///< 已经执行过几次 + int max_retry; ///< 失败后最多再试几次 + uint64_t timeout_ms; ///< 0 = 不限 + uint64_t backoff_ms; ///< 首次退避 + int64_t not_before_ms; ///< epoch ms;0 = 立刻 +}; + +/// 提交参数。显式构造函数(C++11 聚合初始化资格)。 +struct UVCPP_API uvcpp_job_submit { + uvcpp_job_submit(); + + std::string type; + std::string payload; + std::string concurrency_key; + int max_retry; + uint64_t timeout_ms; + uint64_t backoff_ms; +}; + +struct UVCPP_API uvcpp_job_stats { + uvcpp_job_stats(); + size_t pending; + size_t running; + uint64_t success; + uint64_t failed; + uint64_t cancelled; + uint64_t retried; + uint64_t rejected; +}; + +/// 可插拔持久化。实现要对多线程安全(恢复/写入可能跨循环)。 +class UVCPP_API uvcpp_job_store { + public: + virtual ~uvcpp_job_store() {} + virtual bool load_all(std::vector& out) = 0; + virtual bool save(const uvcpp_job_record& rec) = 0; + virtual bool erase(uint64_t id) = 0; +}; + +/// 进程内实现。队列已经在内存里维护全量记录,这个 store 只是让"重启恢复" +/// 在同一个 store 对象上可测;真正跨进程持久化需要 SQLite/MySQL 适配器。 +class UVCPP_API uvcpp_memory_job_store : public uvcpp_job_store { + public: + uvcpp_memory_job_store(); + ~uvcpp_memory_job_store() override; + bool load_all(std::vector& out) override; + bool save(const uvcpp_job_record& rec) override; + bool erase(uint64_t id) override; + private: + struct impl; + std::unique_ptr impl_; +}; + +/// 交给 handler 的上下文。只在 handler 调用期间有效。 +class UVCPP_API uvcpp_job_context { + public: + uvcpp_job_context(); + ~uvcpp_job_context(); + uvcpp_job_context(const uvcpp_job_context&) = delete; + uvcpp_job_context& operator=(const uvcpp_job_context&) = delete; + + /// 超时 / 被取消(协作式)。 + bool cancelled() const; + const uvcpp_job_record& job() const; + + private: + friend class uvcpp_job_queue; + struct impl; + impl* impl_; +}; + +using uvcpp_job_handler = std::function; +using uvcpp_job_state_cb = std::function; + +/** + * @brief 作业队列。见文件头。 + * + * **析构契约**:析构前必须已经 `stop()` 且 `stats().running == 0` —— 执行中的 + * 线程池回调持有本对象的 `impl`,析构时还在跑就是悬垂访问。 + */ +class UVCPP_API uvcpp_job_queue { + public: + explicit uvcpp_job_queue( + uvcpp_loop* loop, + std::shared_ptr store = nullptr); + ~uvcpp_job_queue(); + + uvcpp_job_queue(const uvcpp_job_queue&) = delete; + uvcpp_job_queue& operator=(const uvcpp_job_queue&) = delete; + + /// 注册处理器。同类型重复注册 = 覆盖。没有处理器时任务失败(不吞)。 + void register_handler(const std::string& type, + const uvcpp_job_handler& handler); + + /// 最大同时执行的作业数(默认 4)。只在没有 RUNNING 时改动才立即生效。 + void set_max_concurrent(size_t n); + /// PENDING 上限;0 = 不限(默认)。达到上限时 submit 拒绝。 + void set_max_queue(size_t n); + + /// 提交。成功返回 id(>0);失败返回 0 并写 @p err。 + uint64_t submit(const uvcpp_job_submit& req, std::string* err = nullptr); + + /// 取消。PENDING 直接 CANCELLED;RUNNING 只置协作标志。未知 id 返回 -1。 + int cancel(uint64_t id); + + /// 恢复 store 里的记录并开始调度(幂等)。 + void start(); + /// 停止调度(在跑的允许跑完;不再启动新的)。 + void stop(); + + /// 观察状态变化(loop 线程)。**回调里不要 submit 大批任务**:会重入调度。 + void on_state(const uvcpp_job_state_cb& cb); + + /// 查快照(loop 线程)。未知 id 返回 nullptr。 + const uvcpp_job_record* find(uint64_t id) const; + + /// 丢弃所有已完成(SUCCESS/FAILED/CANCELLED)的记录;返回丢弃数。 + size_t clear_finished(); + + uvcpp_job_stats stats() const; + + private: + struct impl; + impl* impl_; +}; + +} // namespace uvcpp + +#endif // SRC_UVCPP_UVCPP_JOB_QUEUE_H diff --git a/tests/functional/job_queue_func.cpp b/tests/functional/job_queue_func.cpp new file mode 100644 index 00000000..20058273 --- /dev/null +++ b/tests/functional/job_queue_func.cpp @@ -0,0 +1,317 @@ +/** + * @file tests/functional/job_queue_func.cpp + * @brief 作业队列:执行、并发键、重试退避、超时、取消、队列满、重启恢复。 + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include +#include +#include + +using namespace uvcpp; + +namespace { + +int g_fail = 0; + +void check(bool ok, const std::string& what) { + if (ok) { + std::cout << " [ ok ] " << what << std::endl; + } else { + ++g_fail; + std::cerr << " [FAIL] " << what << std::endl; + } +} + +bool pump(uvcpp_loop& loop, const std::function& pred, int ms) { + const std::chrono::steady_clock::time_point deadline = + std::chrono::steady_clock::now() + std::chrono::milliseconds(ms); + // 用 NOWAIT + 1ms 睡眠:作业在 worker 上跑时循环里可能**没有任何事件**, + // `UV_RUN_ONCE` 会一直阻塞到 after_work(长任务就是永远),主线程看不到 + // worker 侧的进展。NOWAIT 轮询保证谓词(往往是原子量)能及时被观察到。 + while (!pred() && std::chrono::steady_clock::now() < deadline) { + loop.run(UV_RUN_NOWAIT); + if (!pred()) std::this_thread::sleep_for(std::chrono::milliseconds(1)); + } + for (int i = 0; i < 5; ++i) loop.run(UV_RUN_NOWAIT); + return pred(); +} + +void test_basic() { + std::cout << "-- 执行 / 状态 / 统计 --" << std::endl; + uvcpp_loop loop; + std::atomic runs(0); + { + uvcpp_job_queue q(&loop); + q.register_handler("ok", [&](uvcpp_job_context&) { ++runs; }); + q.set_max_concurrent(2); + q.start(); + std::string err; + for (int i = 0; i < 3; ++i) { + uvcpp_job_submit s; + s.type = "ok"; + check(q.submit(s, &err) != 0, "提交成功"); + } + check(pump(loop, [&] { return runs.load() == 3; }, 3000), "三个都跑了"); + check(pump(loop, [&] { return q.stats().success == 3; }, 3000), + "stats.success = 3"); + check(q.stats().pending == 0 && q.stats().running == 0, "队列清空"); + check(q.clear_finished() == 3, "clear_finished 清掉 3 条"); + check(q.stats().success == 0, "清掉后统计归零"); + } +} + +void test_concurrency_key() { + std::cout << "-- concurrency_key 串行 --" << std::endl; + uvcpp_loop loop; + std::atomic active(0); + std::atomic max_active(0); + std::atomic done(0); + { + uvcpp_job_queue q(&loop); + q.set_max_concurrent(4); + q.register_handler("work", [&](uvcpp_job_context&) { + const int a = active.fetch_add(1) + 1; + int m = max_active.load(); + while (a > m && !max_active.compare_exchange_weak(m, a)) { + } + std::this_thread::sleep_for(std::chrono::milliseconds(30)); + active.fetch_sub(1); + ++done; + }); + q.start(); + for (int i = 0; i < 5; ++i) { + uvcpp_job_submit s; + s.type = "work"; + s.concurrency_key = "k"; + q.submit(s); + } + check(pump(loop, [&] { return done.load() == 5; }, 5000), "五个都完成"); + check(max_active.load() == 1, "同一 key 从不同时执行(实际最大 " + + std::to_string(max_active.load()) + ")"); + } +} + +void test_retry_and_fail() { + std::cout << "-- 重试 / 耗尽 --" << std::endl; + uvcpp_loop loop; + std::atomic attempts(0); + std::atomic good_done(0); + { + uvcpp_job_queue q(&loop); + q.register_handler("flaky", [&](uvcpp_job_context& ctx) { + ++attempts; + if (ctx.job().attempt < 3) throw std::runtime_error("boom"); + }); + q.register_handler("always_fail", [](uvcpp_job_context&) { + throw std::runtime_error("nope"); + }); + q.start(); + + uvcpp_job_submit s; + s.type = "flaky"; + s.max_retry = 2; + s.backoff_ms = 10; + s.timeout_ms = 0; + const uint64_t id = q.submit(s); + check(pump(loop, [&] { + const uvcpp_job_record* r = q.find(id); + return r != nullptr && + r->state == static_cast(uvcpp_job_state::SUCCESS); + }, 5000), + "失败两次后第三次成功"); + check(q.find(id)->attempt == 3, "attempt = 3"); + check(q.stats().retried == 2, "retried = 2"); + (void)good_done; + + uvcpp_job_submit f; + f.type = "always_fail"; + f.max_retry = 1; + f.backoff_ms = 10; + const uint64_t fid = q.submit(f); + check(pump(loop, [&] { + const uvcpp_job_record* r = q.find(fid); + return r != nullptr && + r->state == static_cast(uvcpp_job_state::FAILED); + }, 5000), + "重试耗尽后 FAILED"); + check(q.find(fid)->attempt == 2, "失败作业尝试了 2 次"); + } +} + +void test_cancel_and_timeout() { + std::cout << "-- 取消 / 超时 --" << std::endl; + uvcpp_loop loop; + std::atomic slow_started(0); + std::atomic victim_ran(0); + { + uvcpp_job_queue q(&loop); + q.set_max_concurrent(1); + std::atomic block_release(false); + q.register_handler("block", [&](uvcpp_job_context& ctx) { + ++slow_started; + while (!block_release.load() && !ctx.cancelled()) { + std::this_thread::sleep_for(std::chrono::milliseconds(5)); + } + }); + q.register_handler("victim", [&](uvcpp_job_context&) { ++victim_ran; }); + q.register_handler("forever", [](uvcpp_job_context& ctx) { + while (!ctx.cancelled()) { + std::this_thread::sleep_for(std::chrono::milliseconds(5)); + } + }); + q.register_handler("slow", [](uvcpp_job_context& ctx) { + while (!ctx.cancelled()) { + std::this_thread::sleep_for(std::chrono::milliseconds(5)); + } + }); + q.start(); + + uvcpp_job_submit b; b.type = "block"; + q.submit(b); + check(pump(loop, [&] { return slow_started.load() == 1; }, 3000), + "占位任务在跑"); + + uvcpp_job_submit v; v.type = "victim"; + const uint64_t vid = q.submit(v); + check(q.cancel(vid) == 0, "取消 PENDING 作业"); + check(q.find(vid)->state == static_cast(uvcpp_job_state::CANCELLED), + "PENDING → CANCELLED"); + + // 放行占位任务,确认被取消的 victim 也不会再跑(它在 PENDING 时已被摘掉)。 + block_release.store(true); + check(pump(loop, [&] { return q.stats().running == 0; }, 3000), + "占位任务结束"); + + uvcpp_job_submit f; f.type = "forever"; + const uint64_t fid = q.submit(f); + check(pump(loop, [&] { + const uvcpp_job_record* r = q.find(fid); + return r != nullptr && + r->state == static_cast(uvcpp_job_state::RUNNING); + }, 3000), + "长任务开始"); + check(q.cancel(fid) == 0, "取消 RUNNING 作业"); + check(pump(loop, [&] { + const uvcpp_job_record* r = q.find(fid); + return r != nullptr && + r->state == static_cast(uvcpp_job_state::CANCELLED); + }, 3000), + "RUNNING → CANCELLED(协作式)"); + + uvcpp_job_submit t; t.type = "slow"; t.timeout_ms = 30; + const uint64_t tid = q.submit(t); + check(pump(loop, [&] { + const uvcpp_job_record* r = q.find(tid); + return r != nullptr && + r->state == static_cast(uvcpp_job_state::FAILED); + }, 3000), + "超时后 FAILED"); + check(q.find(tid)->error == "timeout", "错误标成 timeout"); + check(q.stats().cancelled >= 2, "cancelled 统计 ≥ 2"); + check(victim_ran.load() == 0, "被取消的 PENDING 任务从未执行"); + } +} + +void test_queue_full() { + std::cout << "-- 队列满 --" << std::endl; + uvcpp_loop loop; + std::atomic started(0); + { + uvcpp_job_queue q(&loop); + q.set_max_concurrent(1); + q.set_max_queue(1); + std::atomic hold_release(false); + q.register_handler("hold", [&](uvcpp_job_context& ctx) { + ++started; + while (!hold_release.load() && !ctx.cancelled()) { + std::this_thread::sleep_for(std::chrono::milliseconds(5)); + } + }); + q.start(); + uvcpp_job_submit s; s.type = "hold"; + check(q.submit(s) != 0, "第一个进 RUNNING"); + check(pump(loop, [&] { return started.load() == 1; }, 3000), "它在跑"); + uvcpp_job_submit s2; s2.type = "hold"; + check(q.submit(s2) != 0, "第二个进 PENDING(队列上限 1)"); + uvcpp_job_submit s3; s3.type = "hold"; + std::string err; + check(q.submit(s3, &err) == 0, "第三个被拒绝"); + check(err.find("满") != std::string::npos, "错误说明队列已满"); + check(q.stats().rejected == 1, "rejected = 1"); + hold_release.store(true); + pump(loop, [&] { return q.stats().running == 0; }, 3000); + } +} + +void test_recovery() { + std::cout << "-- 重启恢复 --" << std::endl; + uvcpp_loop loop; + std::shared_ptr store = + std::make_shared(); + uvcpp_job_record crashed; + crashed.id = 7; + crashed.type = "recovered"; + crashed.state = static_cast(uvcpp_job_state::RUNNING); + crashed.attempt = 1; + store->save(crashed); + + std::atomic recovered_done(0); + std::atomic release(false); + { + uvcpp_job_queue q(&loop, store); + q.set_max_concurrent(1); + q.register_handler("block", [&](uvcpp_job_context&) { + while (!release.load()) { + std::this_thread::sleep_for(std::chrono::milliseconds(5)); + } + }); + q.register_handler("recovered", [&](uvcpp_job_context&) { + ++recovered_done; + }); + uvcpp_job_submit b; b.type = "block"; + q.submit(b); + q.start(); + check(pump(loop, [&] { + const uvcpp_job_record* r = q.find(7); + return r != nullptr && + r->state == static_cast(uvcpp_job_state::PENDING); + }, 3000), + "RUNNING 记录恢复成 PENDING"); + check(q.find(7)->error.find("recovered") != std::string::npos, + "记录里标了 recovered after restart"); + release.store(true); + check(pump(loop, [&] { return recovered_done.load() == 1; }, 3000), + "恢复后的任务被执行"); + check(q.find(7)->state == static_cast(uvcpp_job_state::SUCCESS), + "恢复任务成功"); + } +} + +} // namespace + +int main() { + std::cout << "[job_queue] start" << std::endl; + test_basic(); + test_concurrency_key(); + test_retry_and_fail(); + test_cancel_and_timeout(); + test_queue_full(); + test_recovery(); + if (g_fail == 0) { + std::cout << "[job_queue] ALL PASS" << std::endl; + return 0; + } + std::cerr << "[job_queue] " << g_fail << " check(s) failed" << std::endl; + return 1; +}