Support multi images for vlm benchmarking in samples and llm_bench

samples/cpp/text_generation/benchmark_genai.cpp

@@ -3,13 +3,28 @@
 
 #include "openvino/genai/llm_pipeline.hpp"
 #include <cxxopts.hpp>
+#include <fstream>
+#include <sstream>
+
+std::string read_prompt(const std::string& file_path) {
+    std::string prompt;
+    std::ifstream file(file_path);
+    if (file.is_open()) {
+        std::stringstream buffer;
+        buffer << file.rdbuf();
+        prompt = buffer.str();
+        file.close();        
+    }
+    return prompt;
+}
 
 int main(int argc, char* argv[]) try {
     cxxopts::Options options("benchmark_vanilla_genai", "Help command");
 
     options.add_options()
     ("m,model", "Path to model and tokenizers base directory", cxxopts::value<std::string>())
     ("p,prompt", "Prompt", cxxopts::value<std::string>()->default_value("The Sky is blue because"))
+    ("pf,promptfile", "Prompt from file")
     ("nw,num_warmup", "Number of warmup iterations", cxxopts::value<size_t>()->default_value(std::to_string(1)))
     ("n,num_iter", "Number of iterations", cxxopts::value<size_t>()->default_value(std::to_string(3)))
     ("mt,max_new_tokens", "Maximal number of new tokens", cxxopts::value<size_t>()->default_value(std::to_string(20)))
@@ -36,10 +51,18 @@ int main(int argc, char* argv[]) try {
     size_t num_warmup = result["num_warmup"].as<size_t>();
     size_t num_iter = result["num_iter"].as<size_t>();
 
+    if (result.count("promptfile")) {
+        prompt = read_prompt(result["promptfile"].as<std::string>());
+    }
+
     ov::genai::GenerationConfig config;
     config.max_new_tokens = result["max_new_tokens"].as<size_t>();
 
-    ov::genai::LLMPipeline pipe(models_path, device);
+    ov::genai::SchedulerConfig scheduler_config;
+    scheduler_config.enable_prefix_caching = false;
+    scheduler_config.max_num_batched_tokens = 2147483647;
+
+    ov::genai::LLMPipeline pipe(models_path, device, ov::genai::scheduler_config(scheduler_config));
 
     for (size_t i = 0; i < num_warmup; i++)
         pipe.generate(prompt, config);

samples/cpp/visual_language_chat/benchmark_vlm.cpp

@@ -3,10 +3,23 @@
 
 #include <cxxopts.hpp>
 #include <filesystem>
+#include <sstream>
+#include <iostream>
 
 #include "load_image.hpp"
 #include <openvino/genai/visual_language/pipeline.hpp>
 
+std::vector<ov::Tensor> parse_all_images(const std::string &input) {
+    std::vector<ov::Tensor> images;
+    std::stringstream ss(input);
+    std::string image_path;
+    while (std::getline(ss, image_path, ';')) {
+        ov::Tensor image = utils::load_image(image_path);
+        images.push_back(image);
+        std::cout << "input image:" << image_path << std::endl;
+    }
+    return images;
+}
 
 int main(int argc, char* argv[]) try {
     cxxopts::Options options("benchmark_vlm", "Help command");
@@ -41,20 +54,26 @@ int main(int argc, char* argv[]) try {
     std::string device = result["device"].as<std::string>();
     size_t num_warmup = result["num_warmup"].as<size_t>();
     size_t num_iter = result["num_iter"].as<size_t>();
-    ov::Tensor image = utils::load_image(image_path);
+    std::vector<ov::Tensor> images;
+    images = parse_all_images(image_path);
 
     ov::genai::GenerationConfig config;
     config.max_new_tokens = result["max_new_tokens"].as<size_t>();
+    config.ignore_eos = true;
+
+    ov::genai::SchedulerConfig scheduler_config;
+    scheduler_config.enable_prefix_caching = false;
+    scheduler_config.max_num_batched_tokens = 2147483647;
 
-    ov::genai::VLMPipeline pipe(models_path, device);
+    ov::genai::VLMPipeline pipe(models_path, device, ov::genai::scheduler_config(scheduler_config));
 
     for (size_t i = 0; i < num_warmup; i++)
-        pipe.generate(prompt, ov::genai::image(image), ov::genai::generation_config(config));
+        pipe.generate(prompt, ov::genai::image(images), ov::genai::generation_config(config));
 
-    auto res = pipe.generate(prompt, ov::genai::image(image), ov::genai::generation_config(config));
+    auto res = pipe.generate(prompt, ov::genai::image(images), ov::genai::generation_config(config));
     auto metrics = res.perf_metrics;
     for (size_t i = 0; i < num_iter - 1; i++) {
-        res = pipe.generate(prompt, ov::genai::image(image), ov::genai::generation_config(config));
+        res = pipe.generate(prompt, ov::genai::image(images), ov::genai::generation_config(config));
         metrics = metrics + res.perf_metrics;
     }
 

tools/llm_bench/llm_bench_utils/model_utils.py

@@ -321,11 +321,15 @@ def init_timestamp(num_iters, prompt_list, prompt_idx_list):
 
 
 def resolve_media_file_path(file_path, prompt_file_path):
-    if not file_path:
-        return file_path
-    if not (file_path.startswith("http://") or file_path.startswith("https://")):
-        return os.path.join(os.path.dirname(prompt_file_path), file_path.replace("./", ""))
-    return file_path
+    paths_ori = file_path.split(';')
+    paths_new = []
+    for path in paths_ori:
+        if not path:
+            continue
+        if not (path.startswith("http://") or path.startswith("https://")):
+            paths_new.append(os.path.join(os.path.dirname(prompt_file_path), path.replace("./", "")))
+    new_file_path = ";".join(paths_new)
+    return new_file_path
 
 
 def get_version_in_format_to_pars(version):

tools/llm_bench/task/visual_language_generation.py

@@ -201,7 +201,9 @@ def run_visual_language_generation_genai(
     inputs = [inputs] if not isinstance(inputs, (list, tuple)) else inputs
     for input_data in inputs:
         if "media" in input_data:
-            images.append(load_image_genai(input_data["media"]))
+            image_paths = input_data["media"].split(';')
+            for path in image_paths:
+                images.append(load_image_genai(path))
         prompts.append(input_data["prompt"])
     if args["output_dir"] is not None and num == 0:
         for bs_index, in_text in enumerate(prompts):
@@ -221,7 +223,10 @@ def run_visual_language_generation_genai(
     if hasattr(gen_config, 'apply_chat_template'):
         gen_config.apply_chat_template = False
     kwargs = {}
-    if len(images) >= 1:
+    if len(images) > 1:
+        # multi images 
+        kwargs["images"] = images
+    elif len(images) == 1:
         kwargs["images"] = images[0]
     start = time.perf_counter()
     generation_result = model.generate(prompts[0], generation_config=gen_config, **kwargs)