Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 27 additions & 0 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -3095,6 +3095,33 @@ if (ENGINE_BUILD_TESTS OR ENGINE_BUILD_EXTENDED_TESTS OR ENGINE_BUILD_MODEL_TEST
add_test(NAME server_parallel_lifecycle_test COMMAND server_parallel_lifecycle_test)
set_tests_properties(server_parallel_lifecycle_test PROPERTIES TIMEOUT 90)

# Exercise the server exception conversion to HTTP responses
get_target_property(server_exception_sources audiocpp_server SOURCES)
list(FILTER server_exception_sources EXCLUDE REGEX "app/server/(main|model_installer|parallel_runtime)\\.cpp$")
add_executable(server_request_exceptions_test
tests/unittests/test_server_request_exception.cpp ${server_exception_sources})
target_include_directories(server_request_exceptions_test PRIVATE
${CMAKE_CURRENT_SOURCE_DIR}/app/server "${CMAKE_CURRENT_BINARY_DIR}/generated")
target_link_libraries(server_request_exceptions_test PRIVATE engine_runtime ggml Threads::Threads)
if(WIN32)
target_link_libraries(server_request_exceptions_test PRIVATE ws2_32)
endif()
add_test(NAME server_request_exceptions_test COMMAND server_request_exceptions_test)
set_tests_properties(server_request_exceptions_test PROPERTIES TIMEOUT 60)

get_target_property(server_parallel_exception_sources audiocpp_server SOURCES)
list(FILTER server_parallel_exception_sources EXCLUDE REGEX "app/server/(main|model_installer|runtime)\\.cpp$")
add_executable(server_parallel_request_exceptions_test
tests/unittests/test_server_parallel_request_exception.cpp ${server_parallel_exception_sources})
target_include_directories(server_parallel_request_exceptions_test PRIVATE
${CMAKE_CURRENT_SOURCE_DIR}/app/server "${CMAKE_CURRENT_BINARY_DIR}/generated")
target_link_libraries(server_parallel_request_exceptions_test PRIVATE engine_runtime ggml Threads::Threads)
if(WIN32)
target_link_libraries(server_parallel_request_exceptions_test PRIVATE ws2_32)
endif()
add_test(NAME server_parallel_request_exceptions_test COMMAND server_parallel_request_exceptions_test)
set_tests_properties(server_parallel_request_exceptions_test PROPERTIES TIMEOUT 60)

add_executable(session_pool_test tests/unittests/test_session_pool.cpp)
target_include_directories(session_pool_test PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include ${CMAKE_CURRENT_SOURCE_DIR}/app/server)
# TaskResult owns JSON values for custom-schema outputs.
Expand Down
4 changes: 2 additions & 2 deletions app/server/parallel_runtime.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -1283,8 +1283,8 @@ HttpResponse ParallelServerState::handle_request(const HttpRequest & request, bo
else {
response = error_response(404, "unknown endpoint: " + request.path, "not_found");
}
} catch (const engine::runtime::CapacityError & ex) {
// The request is too big for the device, which is the caller's to fix --
} catch (const engine::runtime::RequestValidationError & ex) {
// Something is wrong with the request that the caller can fix --
// reporting it as 500 sends them looking for a server fault that is not
// there. Checked before ServerBusyError only because both are
// runtime_error; the two conditions are disjoint.
Expand Down
6 changes: 3 additions & 3 deletions app/server/runtime.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -1274,8 +1274,8 @@ HttpResponse ServerState::handle_request(const HttpRequest & request, bool use_f
else {
response = error_response(404, "unknown endpoint: " + request.path, "not_found");
}
} catch (const engine::runtime::CapacityError & ex) {
// The request is too big for the device, which is the caller's to fix --
} catch (const engine::runtime::RequestValidationError & ex) {
// Something is wrong with the request that the caller can fix --
// reporting it as 500 sends them looking for a server fault that is not
// there. Checked before ServerBusyError only because both are
// runtime_error; the two conditions are disjoint.
Expand Down Expand Up @@ -1939,7 +1939,7 @@ void ServerState::ensure_model_loaded_locked(LoadedModel & model) {
// That is acceptable: the 503 tells the client to retry later.
}
ensure_model_fits_memory(model.config);
auto registry = engine::runtime::make_default_registry();
auto registry = registry_factory_();

engine::runtime::ModelLoadRequest load_request;
load_request.model_path = model.config.path;
Expand Down
7 changes: 7 additions & 0 deletions app/server/runtime.h
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
#include "engine/framework/io/json.h"
#include "engine/framework/runtime/model.h"
#include "engine/framework/runtime/session.h"
#include "engine/framework/runtime/registry.h"

#include <atomic>
#include <cstdint>
Expand Down Expand Up @@ -92,6 +93,12 @@ class ServerState final : public IHttpHandler, public ServerFrontendContext {
void unload();
};

// Controlled runtime tests suppy loaders and barriers without production flags.
friend class ServerRuntimeTestAccess;
std::function<engine::runtime::ModelRegistry()> registry_factory_ = [] {
return engine::runtime::make_default_registry();
};

// Acquire the model's run guard. `request_timeout_ms` is the caller-supplied
// override, clamped by this model's configured ceiling. Throws ServerBusyError
// (-> HTTP 503) once the effective timeout has elapsed.
Expand Down
16 changes: 11 additions & 5 deletions include/engine/framework/runtime/errors.h
Original file line number Diff line number Diff line change
Expand Up @@ -5,13 +5,19 @@

namespace engine::runtime {

// A request has a problem that the client can fix by changing request
// parameters. Distinct from a genuine internal fault: so servers should
// surface it as a client error rather than an opaque 500.
class RequestValidationError : public std::runtime_error {
public:
explicit RequestValidationError(const std::string & message) : std::runtime_error(message) {}
};

// A request the device cannot serve AT THIS SIZE -- e.g. a transcription
// prompt plus audio whose prefill graph does not fit in VRAM. Distinct from a
// genuine internal fault: the caller can fix it by sending less, so servers
// should surface it as a client error rather than an opaque 500.
class CapacityError : public std::runtime_error {
// prompt plus audio whose prefill graph does not fit in VRAM.
class CapacityError : public RequestValidationError {
public:
explicit CapacityError(const std::string & message) : std::runtime_error(message) {}
explicit CapacityError(const std::string & message) : RequestValidationError(message) {}
};

} // namespace engine::runtime
Loading