init

Author: mzegla

demos/common/export_models/README.md

@@ -55,7 +55,9 @@ Some of the exceptions include:
 
 ### Text Generation Models
 
-#### Text Generation CPU Deployment
+When `--target_device` is omitted, no device field is written to the graph configuration and OVMS auto-detects the best available device at startup. Pass `--target_device` explicitly to pin execution to a specific device.
+
+#### Text Generation (Auto-detected Device)
 ```console
 python export_model.py text_generation --source_model meta-llama/Meta-Llama-3-8B-Instruct --weight-format fp16 --kv_cache_precision u8 --config_file_path models/config_all.json --model_repository_path models
 ```

demos/common/export_models/export_model.py

@@ -30,7 +30,7 @@ def add_common_arguments(parser):
     parser.add_argument('--weight-format', default='int8', help='precision of the exported model', dest='precision')
     parser.add_argument('--config_file_path', default='config.json', help='path to the config file', dest='config_file_path')
     parser.add_argument('--overwrite_models', default=False, action='store_true', help='Overwrite the model if it already exists in the models repository', dest='overwrite_models')
-    parser.add_argument('--target_device', default="CPU", help='CPU, GPU, NPU or HETERO, default is CPU', dest='target_device')
+    parser.add_argument('--target_device', default=None, help='CPU, GPU, NPU or HETERO. When not specified the server will auto-detect the best available device at startup.', dest='target_device')
     parser.add_argument('--ov_cache_dir', default=None, help='Folder path for compilation cache to speedup initialization time', dest='ov_cache_dir')
     parser.add_argument('--extra_quantization_params', required=False, help='Add advanced quantization parameters. Check optimum-intel documentation. Example: "--sym --group-size -1 --ratio 1.0 --awq --scale-estimation --dataset wikitext2"', dest='extra_quantization_params')
 
@@ -240,12 +240,14 @@ def add_common_arguments(parser):
           max_num_batched_tokens: {{max_num_batched_tokens}},{% endif %}
           {%- if not dynamic_split_fuse %}
           dynamic_split_fuse: false, {% endif %}
-          max_num_seqs: {% if draft_eagle3_mode %}1{% else %}{{max_num_seqs|default("256", true)}}{% endif %},
-          device: "{{target_device|default("CPU", true)}}",
+          max_num_seqs: {{max_num_seqs|default("256", true)}},
+          {%- if target_device %}
+          device: "{{target_device}}",{% endif %}
           {%- if draft_model_dir_name %}
           # Speculative decoding configuration
           draft_models_path: "./{{draft_model_dir_name}}",
-          draft_device: "{{target_device|default("CPU", true)}}",
+          {%- if target_device %}
+          draft_device: "{{target_device}}",{% endif %}
           draft_eagle3_mode: {{draft_eagle3_mode|default(false)}},{% endif %}
           {%- if reasoning_parser %}
           reasoning_parser: "{{reasoning_parser}}",{% endif %}
@@ -458,7 +460,7 @@ def export_text_generation_model(model_repository_path, source_model, model_name
         plugin_config['prompt_lookup'] = True
 
     # Additional plugin properties for HETERO
-    if "HETERO" in task_parameters['target_device']:
+    if task_parameters['target_device'] and "HETERO" in task_parameters['target_device']:
         plugin_config['MODEL_DISTRIBUTION_POLICY'] = 'PIPELINE_PARALLEL'
 
     if task_parameters['target_device'] == 'NPU':

demos/continuous_batching/accuracy/README.md

@@ -25,9 +25,9 @@ Example of LLM and VLM models deployment is documented in other demos like
 ## Running the tests for LLM models
 
 ```text
-lm-eval --model local-chat-completions --tasks gsm8k --model_args model=meta-llama/Meta-Llama-3.1-8B-Instruct,base_url=http://localhost:8000/v3/chat/completions,num_concurrent=1,max_retries=3,tokenized_requests=False --verbosity DEBUG  --log_samples --output_path test/ --seed 1 --apply_chat_template --limit 100
+lm-eval --model local-chat-completions --tasks gsm8k --model_args model=meta-llama/Meta-Llama-3.1-8B-Instruct,base_url=http://localhost:8000/v1/chat/completions,num_concurrent=1,max_retries=3,tokenized_requests=False --verbosity DEBUG  --log_samples --output_path test/ --seed 1 --apply_chat_template --limit 100
 
-local-chat-completions ({'model': 'meta-llama/Meta-Llama-3.1-8B-Instruct', 'base_url': 'http://localhost:8000/v3/chat/completions', 'num_concurrent': 10, 'max_retries': 3, 'tokenized_requests': False}), gen_kwargs: ({}), limit: 100.0, num_fewshot: None, batch_size: 1
+local-chat-completions ({'model': 'meta-llama/Meta-Llama-3.1-8B-Instruct', 'base_url': 'http://localhost:8000/v1/chat/completions', 'num_concurrent': 10, 'max_retries': 3, 'tokenized_requests': False}), gen_kwargs: ({}), limit: 100.0, num_fewshot: None, batch_size: 1
 |Tasks|Version|     Filter     |n-shot|  Metric   |   |Value|   |Stderr|
 |-----|------:|----------------|-----:|-----------|---|----:|---|-----:|
 |gsm8k|      3|flexible-extract|     5|exact_match|↑  | 0.62|±  |0.0488|
@@ -37,9 +37,9 @@ local-chat-completions ({'model': 'meta-llama/Meta-Llama-3.1-8B-Instruct', 'base
 While testing the non chat model and `completion` endpoint, the command would look like this:
 
 ```text
-lm-eval --model local-completions --tasks gsm8k --model_args model=meta-llama/Meta-Llama-3.1-8B,base_url=http://localhost:8000/v3/completions,num_concurrent=1,max_retries=3,tokenized_requests=False --verbosity DEBUG  --log_samples --output_path results/ --seed 1 --limit 100
+lm-eval --model local-completions --tasks gsm8k --model_args model=meta-llama/Meta-Llama-3.1-8B,base_url=http://localhost:8000/v1/completions,num_concurrent=1,max_retries=3,tokenized_requests=False --verbosity DEBUG  --log_samples --output_path results/ --seed 1 --limit 100
 
-local-completions ({'model': 'meta-llama/Meta-Llama-3.1-8B', 'base_url': 'http://localhost:8000/v3/completions', 'num_concurrent': 10, 'max_retries': 3, 'tokenized_requests': False}), gen_kwargs: ({}), limit: 100.0, num_fewshot: None, batch_size: 1
+local-completions ({'model': 'meta-llama/Meta-Llama-3.1-8B', 'base_url': 'http://localhost:8000/v1/completions', 'num_concurrent': 10, 'max_retries': 3, 'tokenized_requests': False}), gen_kwargs: ({}), limit: 100.0, num_fewshot: None, batch_size: 1
 |Tasks|Version|     Filter     |n-shot|  Metric   |   |Value|   |Stderr|
 |-----|------:|----------------|-----:|-----------|---|----:|---|-----:|
 |gsm8k|      3|flexible-extract|     5|exact_match|↑  | 0.43|±  |0.0498|
@@ -49,19 +49,19 @@ local-completions ({'model': 'meta-llama/Meta-Llama-3.1-8B', 'base_url': 'http:/
 Other examples are below:
 
 ```text
-lm-eval --model local-chat-completions --tasks leaderboard_ifeval --model_args model=meta-llama/Meta-Llama-3.1-8B-Instruct,base_url=http://localhost:8000/v3/chat/completions,num_concurrent=10,max_retries=3,tokenized_requests=False --verbosity DEBUG --log_samples --output_path test/ --seed 1 --limit 100 --apply_chat_template  
+lm-eval --model local-chat-completions --tasks leaderboard_ifeval --model_args model=meta-llama/Meta-Llama-3.1-8B-Instruct,base_url=http://localhost:8000/v1/chat/completions,num_concurrent=10,max_retries=3,tokenized_requests=False --verbosity DEBUG --log_samples --output_path test/ --seed 1 --limit 100 --apply_chat_template  
 ```
 
 ```text
-lm-eval --model local-completions --tasks wikitext --model_args model=meta-llama/Meta-Llama-3.1-8B,base_url=http://localhost:8000/v3/completions,num_concurrent=10,max_retries=3,tokenized_requests=False --verbosity DEBUG --log_samples --output_path test/ --seed 1 --limit 100
+lm-eval --model local-completions --tasks wikitext --model_args model=meta-llama/Meta-Llama-3.1-8B,base_url=http://localhost:8000/v1/completions,num_concurrent=10,max_retries=3,tokenized_requests=False --verbosity DEBUG --log_samples --output_path test/ --seed 1 --limit 100
 ```
 
 ## Running the tests for VLM models
 
 Use [lmms-eval project](https://github.com/EvolvingLMMs-Lab/lmms-eval) - mme and mmmu_val tasks. 
 
 ```text
-export OPENAI_BASE_URL=http://localhost:8000/v3
+export OPENAI_BASE_URL=http://localhost:8000/v1
 export OPENAI_API_KEY="unused"
 git clone https://github.com/EvolvingLMMs-Lab/lmms-eval
 cd lmms-eval
@@ -103,7 +103,7 @@ pip install -e . --extra-index-url "https://download.pytorch.org/whl/cpu"
 ```
 The commands below assumes the models is deployed with the name `ovms-model`. It must match the name set in the `bfcl_eval/constants/model_config.py`.
 ```text
-export OPENAI_BASE_URL=http://localhost:8000/v3
+export OPENAI_BASE_URL=http://localhost:8000/v1
 export CHAT_TEMPLATE_KWARGS='{"enable_thinking":false, "reasoning_effort":"low", "preserve_reasoning":false}'
 
 bfcl generate --model ovms-model --test-category simple_python,multiple,multi_turn_base --temperature 0.0 --num-threads 10 -o --result-dir model_name_dir
@@ -112,7 +112,7 @@ bfcl evaluate --model ovms-model --result-dir model_name_dir
 
 Alternatively, use the model name `ovms-model-stream` to run the tests with stream requests. The results should be the same.
 ```text
-export OPENAI_BASE_URL=http://localhost:8000/v3
+export OPENAI_BASE_URL=http://localhost:8000/v1
 bfcl generate --model ovms-model-stream --test-category simple_python,multiple,multi_turn_base --temperature 0.0 --num-threads 10 -o --result-dir model_name_dir
 bfcl evaluate --model ovms-model-stream --result-dir model_name_dir 
 ```
@@ -127,8 +127,3 @@ cat score/openvino-qwen3-8b-int4-FC/BFCL_v4_simple_python_score.json | head -1
 Those results can be compared with the reference from the [berkeley leaderbaord](https://gorilla.cs.berkeley.edu/leaderboard.html#leaderboard).
 
 ---
-
-> **Note:** The same procedure can be used to validate vLLM component. The only needed change would be updating base_url including replacing `/v3/` with `/v1/`.  
-
-
-

demos/continuous_batching/speculative_decoding/README.md

@@ -90,11 +90,12 @@ models
 
 :::{dropdown} **Deploying with Docker**
 ```bash
-docker run -d --rm -p 8000:8000 -v $(pwd)/models:/workspace:ro openvino/model_server:weekly --rest_port 8000 --config_path /workspace/config.json
+docker run -d --rm $(test -d /dev/dri && echo "--device /dev/dri --group-add $(stat -c '%g' /dev/dri/render* | head -n1)") \
+  -p 8000:8000 -v $(pwd)/models:/workspace:ro openvino/model_server:weekly \
+  --rest_port 8000 --config_path /workspace/config.json
 ```
 
-Running above command starts the container with no accelerators support. 
-To deploy on devices other than CPU, change `target_device` parameter in `export_model.py` call and follow [AI accelerators guide](../../../docs/accelerators.md) for additionally required docker parameters.
+OVMS auto-detects the best available device at startup. To target a specific device explicitly, pass `--target_device GPU` (or `NPU`, `HETERO:GPU,CPU`, etc.) to `export_model.py` and follow the [AI accelerators guide](../../../docs/accelerators.md) for additionally required docker parameters.
 :::
 
 :::{dropdown} **Deploying on Bare Metal**
@@ -119,7 +120,7 @@ Let's check how the deployed model is doing by running performance test. For tha
 
 Install vLLM and download sonnet dataset: 
 ```bash
-pip install vllm --extra-index-url https://wheels.vllm.ai/nightly/cpu
+pip install vllm --index-url https://wheels.vllm.ai/nightly/cpu --extra-index-url https://pypi.org/simple
 curl https://raw.githubusercontent.com/vllm-project/vllm/refs/heads/main/benchmarks/sonnet.txt -o sonnet.txt
 ```
 
@@ -287,8 +288,7 @@ models
 docker run -d --rm -p 8000:8000 -v $(pwd)/models:/workspace:ro openvino/model_server:latest --rest_port 8000 --config_path /workspace/config.json
 ```
 
-Running above command starts the container with no accelerators support. 
-To deploy on devices other than CPU, change `target_device` parameter in `export_model.py` call and follow [AI accelerators guide](../../../docs/accelerators.md) for additionally required docker parameters.
+OVMS auto-detects the best available device at startup. To target a specific device explicitly, pass `--target_device GPU` (or `NPU`, `HETERO:GPU,CPU`, etc.) to `export_model.py` and follow the [AI accelerators guide](../../../docs/accelerators.md) for additionally required docker parameters.
 :::
 
 :::{dropdown} **Deploying on Bare Metal**

docs/parameters.md

@@ -147,7 +147,10 @@ Task specific parameters for different tasks (text generation/image generation/e
 | `--enable_prefix_caching`             | `bool`       | Enables algorithm to cache the prompt tokens. Default: true.                                                               |
 | `--max_num_batched_tokens`            | `integer`    | The maximum number of tokens that can be batched together.                                                                 |
 | `--cache_size`                        | `integer`    | KV Cache size in GB. Default: 0 which is a dynamic allocation.                              |
-| `--draft_source_model`                | `string`     | HF model name or path to the local folder with PyTorch or OpenVINO draft model.                                            |
+| `--draft_source_model`                | `string`     | HF model name or path to the local folder with PyTorch or OpenVINO draft model. Mutually exclusive with `--draft_model_path`.  |
+| `--draft_model_path`                  | `string`     | Absolute path to an already-exported OpenVINO draft model directory. Use for local models instead of `--draft_source_model`.   |
+| `--draft_eagle3_mode`                 | `bool`       | Enable EAGLE3 speculative decoding for the draft model. Default: false.                                                        |
+| `--draft_device`                      | `string`     | Device to run the draft model on. Defaults to the same device as the main model.                                              |
 | `--dynamic_split_fuse`                | `bool`       | Enables dynamic split fuse algorithm. Default: true.                                                                       |
 | `--max_prompt_len`                    | `integer`    | Sets NPU specific property for maximum number of tokens in the prompt.                                                     |
 | `--kv_cache_precision`                | `string`     | Reduced kv cache precision to `u8` lowers the cache size consumption. Accepted values: `u8` or empty (default).            |

src/capi_frontend/server_settings.hpp

@@ -122,6 +122,8 @@ struct TextGenGraphSettingsImpl {
     std::optional<uint32_t> maxNumBatchedTokens;
     std::optional<std::string> draftModelDirName;
     bool draftEagle3Mode = false;
+    std::optional<std::string> draftDevice;
+    std::optional<std::string> draftModelPath;
     std::optional<std::string> pipelineType;
     std::optional<std::string> reasoningParser;
     std::optional<std::string> toolParser;

src/graph_export/graph_cli_parser.cpp

@@ -65,10 +65,18 @@ void GraphCLIParser::createOptions() {
             "HF model name or path to the local folder with PyTorch or OpenVINO draft model.",
             cxxopts::value<std::string>(),
             "DRAFT_SOURCE_MODEL")
+        ("draft_model_path",
+            "Absolute path to an already-exported OpenVINO draft model directory (use instead of --draft_source_model for local models).",
+            cxxopts::value<std::string>(),
+            "DRAFT_MODEL_PATH")
         ("draft_eagle3_mode",
             "Enable EAGLE3 speculative decoding mode for the draft model.",
             cxxopts::value<bool>()->default_value("false")->implicit_value("true"),
             "DRAFT_EAGLE3_MODE")
+        ("draft_device",
+            "Device to run the draft model on. Defaults to the same device as the main model.",
+            cxxopts::value<std::string>(),
+            "DRAFT_DEVICE")
         ("dynamic_split_fuse",
             "Dynamic split fuse algorithm enabled. Default true.",
             cxxopts::value<std::string>()->default_value("true"),
@@ -152,9 +160,17 @@ void GraphCLIParser::prepare(OvmsServerMode serverMode, HFSettingsImpl& hfSettin
         if (result->count("draft_source_model")) {
             graphSettings.draftModelDirName = result->operator[]("draft_source_model").as<std::string>();
         }
+        if (result->count("draft_model_path")) {
+            if (result->count("draft_source_model"))
+                throw std::invalid_argument("--draft_model_path and --draft_source_model are mutually exclusive");
+            graphSettings.draftModelPath = result->operator[]("draft_model_path").as<std::string>();
+        }
         if (result->count("draft_eagle3_mode")) {
             graphSettings.draftEagle3Mode = result->operator[]("draft_eagle3_mode").as<bool>();
         }
+        if (result->count("draft_device")) {
+            graphSettings.draftDevice = result->operator[]("draft_device").as<std::string>();
+        }
         if (result->count("pipeline_type")) {
             graphSettings.pipelineType = result->operator[]("pipeline_type").as<std::string>();
         }

src/graph_export/graph_export.cpp

@@ -217,11 +217,20 @@ static Status createTextGenerationGraphTemplate(const std::string& directoryPath
         oss << R"(
             dynamic_split_fuse: false,)";
     }
-    if (graphSettings.draftModelDirName.has_value()) {
+    if (graphSettings.draftModelDirName.has_value() || graphSettings.draftModelPath.has_value()) {
         oss << R"(
             # Speculative decoding configuration)";
-        oss << R"(
+        if (graphSettings.draftModelPath.has_value()) {
+            oss << R"(
+            draft_models_path: ")" << graphSettings.draftModelPath.value() << R"(",)";
+        } else {
+            oss << R"(
             draft_models_path: ")" << GraphExport::getDraftModelDirectoryName(graphSettings.draftModelDirName.value()) << R"(",)";
+        }
+        if (graphSettings.draftDevice.has_value()) {
+            oss << R"(
+            draft_device: ")" << graphSettings.draftDevice.value() << R"(",)";
+        }
         if (graphSettings.draftEagle3Mode) {
             oss << R"(
             draft_eagle3_mode: true,)";