Enable CDPruner

samples/cpp/visual_language_chat/benchmark_vlm.cpp

@@ -20,6 +20,8 @@ int main(int argc, char* argv[]) try {
     ("n,num_iter", "Number of iterations", cxxopts::value<size_t>()->default_value(std::to_string(3)))
     ("mt,max_new_tokens", "Maximal number of new tokens", cxxopts::value<size_t>()->default_value(std::to_string(20)))
     ("d,device", "device", cxxopts::value<std::string>()->default_value("CPU"))
+    ("pr,pruning_ratio", "Percentage of visual tokens to prune when CDPruner is enabled", cxxopts::value<size_t>()->default_value("50"))
+    ("pdm,pruning_debug_mode", "Enable pruning debug mode", cxxopts::value<bool>()->default_value("false"))
     ("h,help", "Print usage");
 
     cxxopts::ParseResult result;
@@ -57,18 +59,34 @@ int main(int argc, char* argv[]) try {
     std::string device = result["device"].as<std::string>();
     size_t num_warmup = result["num_warmup"].as<size_t>();
     size_t num_iter = result["num_iter"].as<size_t>();
+    size_t pruning_ratio = result["pruning_ratio"].as<size_t>();
+    bool pruning_debug_mode = result["pruning_debug_mode"].as<bool>();
     std::vector<ov::Tensor> images = utils::load_images(image_path);
 
     ov::genai::GenerationConfig config;
     config.max_new_tokens = result["max_new_tokens"].as<size_t>();
     config.ignore_eos = true;
+
+    config.pruning_ratio = pruning_ratio;
+    // Configure CDPruner if requested
+    if (pruning_ratio > 0 && pruning_ratio < 100) {
+        std::cout << "[CDPruner] Enabling CDPruner with pruning ratio " << pruning_ratio << "% visual tokens" << std::endl;
+        config.pruning_debug_mode = pruning_debug_mode;
+    }
 
     std::cout << ov::get_openvino_version() << std::endl;
 
+    // Setup cache configuration for CDPruner if needed
+    ov::AnyMap properties = {};
+    if (pruning_ratio > 0 && pruning_ratio < 100) {
+        properties.insert({"ATTENTION_BACKEND", "PA"});
+        std::cout << "[CDPruner] Setting ATTENTION_BACKEND to PA for CDPruner" << std::endl;
+    }
+
     std::unique_ptr<ov::genai::VLMPipeline> pipe;
-    if (device == "NPU")
+    if (device == "NPU") {
         pipe = std::make_unique<ov::genai::VLMPipeline>(models_path, device);
-    else {
+    } else {
         // Setting of Scheduler config will trigger usage of ContinuousBatching pipeline, which is not default for Qwen2VL, Qwen2.5VL, Gemma3 due to accuracy issues.
         ov::genai::SchedulerConfig scheduler_config;
         scheduler_config.enable_prefix_caching = false;

samples/cpp/visual_language_chat/visual_language_chat.cpp

@@ -1,4 +1,4 @@
-// Copyright (C) 2024 Intel Corporation
+// Copyright (C) 2024 Intel Corporation
 // SPDX-License-Identifier: Apache-2.0
 
 #include "load_image.hpp"
@@ -10,25 +10,41 @@ bool print_subword(std::string&& subword) {
 }
 
 int main(int argc, char* argv[]) try {
-    if (argc < 3 || argc > 4) {
-        throw std::runtime_error(std::string{"Usage "} + argv[0] + " <MODEL_DIR> <IMAGE_FILE OR DIR_WITH_IMAGES> <DEVICE>");
+    if (3 > argc || argc > 6) {
+        throw std::runtime_error(std::string{"Usage: "} + argv[0] + " <MODEL_DIR> <IMAGE_FILE> [<DEVICE>] [<PRUNING_RATIO>] [<PRUNING_DEBUG_MODE>]");
     }
 
-    std::vector<ov::Tensor> rgbs = utils::load_images(argv[2]);
+    std::string model_dir = argv[1];
+    std::string image_file = argv[2];
+    std::string device = argc > 3 ? argv[3] : "CPU";
+    size_t pruning_ratio = argc > 4 ? std::stoul(argv[4]) : 0;  // 0 means disabled
+    bool pruning_debug_mode = argc > 5 ? (std::string(argv[5]) == "true" || std::string(argv[5]) == "1") : false;
+
+    std::vector<ov::Tensor> rgbs = utils::load_images(image_file);
 
     // GPU and NPU can be used as well.
     // Note: If NPU is selected, only language model will be run on NPU
-    std::string device = (argc == 4) ? argv[3] : "CPU";
     ov::AnyMap enable_compile_cache;
     if (device == "GPU") {
-        // Cache compiled models on disk for GPU to save time on the
-        // next run. It's not beneficial for CPU.
         enable_compile_cache.insert({ov::cache_dir("vlm_cache")});
     }
-    ov::genai::VLMPipeline pipe(argv[1], device, enable_compile_cache);
 
+    if (pruning_ratio > 0) {
+        enable_compile_cache.insert({"ATTENTION_BACKEND", "PA"});
+        std::cout << "[CDPruner] Setting ATTENTION_BACKEND to PA" << std::endl;
+    }
+
+    // Initialize VLMPipeline with cache configuration if needed
+    ov::genai::VLMPipeline pipe(model_dir, device, enable_compile_cache);
+
     ov::genai::GenerationConfig generation_config;
     generation_config.max_new_tokens = 100;
+    generation_config.pruning_ratio = pruning_ratio;
+    // Configure CDPruner if requested
+    if (pruning_ratio > 0) {
+        std::cout << "[CDPruner] Enabling CDPruner with " << pruning_ratio << "% visual token pruning" << std::endl;
+        generation_config.pruning_debug_mode = pruning_debug_mode;
+    }
 
     std::string prompt;
 

samples/python/visual_language_chat/benchmark_vlm.py

@@ -42,6 +42,9 @@ def main():
     parser.add_argument("-n", "--num_iter", type=int, default=2, help="Number of iterations")
     parser.add_argument("-mt", "--max_new_tokens", type=int, default=20, help="Maximal number of new tokens")
     parser.add_argument("-d", "--device", type=str, default="CPU", help="Device")
+    parser.add_argument("--pruning_ratio", type=int, default=0, help="Percentage of visual tokens to prune (0 to disable)")
+    parser.add_argument("--pruning_debug_mode", action="store_true", help="Enable debugging mode for pruning")
+    parser.add_argument("--relevance_weight", type=float, help="Relevance weight for the model")
 
     args = parser.parse_args()
 
@@ -68,6 +71,14 @@ def main():
 
     config = ov_genai.GenerationConfig()
     config.max_new_tokens = args.max_new_tokens
+    config.pruning_ratio = args.pruning_ratio if args.pruning_ratio is not None else 0
+    print(f'CDPruner config: Pruning ratio - {config.pruning_ratio}% (0 means disabled)')
+    if config.pruning_ratio > 0:
+        if args.relevance_weight is not None:
+            config.relevance_weight = args.relevance_weight
+        if args.pruning_debug_mode:
+            config.pruning_debug_mode = args.pruning_debug_mode
+        print(f'CDPruner config: Pruning debug mode - {config.pruning_debug_mode}')
 
     if device == "NPU":
         pipe = ov_genai.VLMPipeline(models_path, device)

src/cpp/CMakeLists.txt

@@ -9,6 +9,19 @@ list(APPEND SOURCE_FILES "${CMAKE_CURRENT_BINARY_DIR}/version.cpp")
 
 include(FetchContent)
 
+# OpenCL support for CDPruner DPP acceleration - reuse OpenVINO's ENABLE_SYSTEM_OPENCL
+if(ENABLE_SYSTEM_OPENCL)
+    # Try to find OpenCL since ENABLE_SYSTEM_OPENCL is ON
+    find_package(OpenCL QUIET)
+    if(TARGET OpenCL::OpenCL)
+        message(STATUS "OpenCL found via OpenVINO configuration - enabling CDPruner DPP acceleration")
+    else()
+        message(STATUS "ENABLE_SYSTEM_OPENCL is ON but OpenCL::OpenCL target not found - CDPruner will use CPU-only DPP")
+    endif()
+else()
+    message(STATUS "ENABLE_SYSTEM_OPENCL is OFF - CDPruner will use CPU-only DPP implementation")
+endif()
+
 if(NOT TARGET nlohmann_json)
     FetchContent_Declare(nlohmann_json
         URL https://github.com/nlohmann/json/archive/refs/tags/v3.11.3.tar.gz
@@ -143,6 +156,12 @@ target_include_directories(${TARGET_NAME_OBJ} SYSTEM PRIVATE "${safetensors.h_SO
 
 target_link_libraries(${TARGET_NAME_OBJ} PRIVATE openvino::runtime openvino::threading nlohmann_json::nlohmann_json minja)
 
+# Add OpenCL support if enabled via OpenVINO configuration
+if(ENABLE_SYSTEM_OPENCL AND TARGET OpenCL::OpenCL)
+    target_compile_definitions(${TARGET_NAME_OBJ} PRIVATE ENABLE_OPENCL_DPP)
+    target_link_libraries(${TARGET_NAME_OBJ} PRIVATE OpenCL::OpenCL)
+endif()
+
 target_compile_features(${TARGET_NAME_OBJ} PRIVATE cxx_std_17)
 
 target_compile_definitions(${TARGET_NAME_OBJ} PRIVATE openvino_genai_EXPORTS)
@@ -152,6 +171,16 @@ if(MSVC)
     target_compile_options(${TARGET_NAME_OBJ} PRIVATE "/bigobj")
 endif()
 
+# Add native CPU optimization for SIMD instructions
+if(CMAKE_CXX_COMPILER_ID MATCHES "GNU|Clang")
+    # Force AVX2 only (disable AVX512)
+    target_compile_options(${TARGET_NAME_OBJ} PRIVATE "-mavx2" "-mno-avx512f")
+elseif(MSVC)
+    if(CMAKE_SYSTEM_PROCESSOR MATCHES "AMD64|x86_64")
+        target_compile_options(${TARGET_NAME_OBJ} PRIVATE "/arch:AVX2")
+    endif()
+endif()
+
 set_target_properties(${TARGET_NAME_OBJ} PROPERTIES POSITION_INDEPENDENT_CODE ON)
 
 # Shared library
@@ -169,6 +198,11 @@ target_include_directories(${TARGET_NAME} INTERFACE "$<INSTALL_INTERFACE:runtime
 
 target_link_libraries(${TARGET_NAME} PUBLIC openvino::runtime PRIVATE openvino::threading nlohmann_json::nlohmann_json minja ${CMAKE_DL_LIBS})
 
+# Add OpenCL support if enabled via OpenVINO configuration
+if(ENABLE_SYSTEM_OPENCL AND TARGET OpenCL::OpenCL)
+    target_link_libraries(${TARGET_NAME} PRIVATE OpenCL::OpenCL)
+endif()
+
 if(ENABLE_XGRAMMAR)
     target_link_libraries(${TARGET_NAME} PRIVATE xgrammar)
 endif()

src/cpp/include/openvino/genai/generation_config.hpp

@@ -271,6 +271,11 @@ operator|(const StructuredOutputConfig::CompoundGrammar& lhs,
  * @param top_k the number of highest probability vocabulary tokens to keep for top-k-filtering.
  * @param rng_seed initializes random generator.
  *
+ * CDPruner configuration:
+ * @param pruning_ratio the percentage of visual tokens to prune (0-100). Set to 0 to disable pruning.
+ * @param relevance_weight the weight of relevance for visual tokens.
+ * @param pruning_debug_mode whether to enable pruning debug mode.
+ *
  * Assisting generation parameters:
  * @param assistant_confidence_threshold the lower token probability of candidate to be validated by main model in case of dynamic strategy candidates number update.
  * @param num_assistant_tokens the defined candidates number to be generated by draft model/prompt lookup in case of static strategy candidates number update.
@@ -321,6 +326,11 @@ class OPENVINO_GENAI_EXPORTS GenerationConfig {
     bool do_sample = false;
     size_t rng_seed = 0;
 
+    // CDPruner config
+    size_t pruning_ratio = 0;  // 0 means disabled, 1-100 means percentage to prune
+    float relevance_weight = 0.5f;
+    bool pruning_debug_mode = false;
+
     // Assisting generation parameters
     float assistant_confidence_threshold = 0.f;
     size_t num_assistant_tokens = 0;
@@ -392,6 +402,11 @@ static constexpr ov::Property<float> repetition_penalty{"repetition_penalty"};
 static constexpr ov::Property<int64_t> eos_token_id{"eos_token_id"};
 static constexpr ov::Property<float> presence_penalty{"presence_penalty"};
 static constexpr ov::Property<float> frequency_penalty{"frequency_penalty"};
+
+static constexpr ov::Property<size_t> pruning_ratio{"pruning_ratio"};
+static constexpr ov::Property<float> relevance_weight{"relevance_weight"};
+static constexpr ov::Property<bool> pruning_debug_mode{"pruning_debug_mode"};
+
 extern OPENVINO_GENAI_EXPORTS ov::Property<size_t> rng_seed;
 
 static constexpr ov::Property<float> assistant_confidence_threshold{"assistant_confidence_threshold"};

src/cpp/src/continuous_batching/pipeline_base.cpp

@@ -164,6 +164,12 @@ ContinuousBatchingPipeline::IContinuousBatchingPipeline::generate(
     std::vector<VLMPerfMetrics> vlm_perf_metrics(prompts.size());
     std::vector<EncodedImage> encoded_images = {};
 
+    const auto& generation_config = sampling_params[0];
+    // Set visual token pruning configuration
+    m_inputs_embedder->set_visual_token_pruning_config(generation_config.pruning_ratio,
+                                                       generation_config.relevance_weight,
+                                                       generation_config.pruning_debug_mode);
+
     if (m_is_chat_conversation) {
         OPENVINO_ASSERT(1 == prompts.size(), "Can't chat with multiple prompts");
         const auto& rgbs = rgbs_vector[0];