[GGUF] Serialize Generated OV Model for Faster LLMPipeline Init

src/cpp/src/gguf_utils/gguf_modeling.cpp

@@ -11,6 +11,7 @@
 #include <openvino/openvino.hpp>
 #include "openvino/runtime/core.hpp"
 #include "openvino/opsets/opset13.hpp"
+#include "openvino/pass/serialize.hpp"
 
 #include "gguf_utils/building_blocks.hpp"
 #include "gguf_utils/gguf_modeling.hpp"
@@ -152,24 +153,47 @@ std::shared_ptr<ov::Model> create_language_model(
 
 } // namespace
 
-std::shared_ptr<ov::Model> create_from_gguf(const std::string& model_path) {
+void save_openvino_model(const std::shared_ptr<ov::Model>& model, const std::string& save_path, bool compress_to_fp16) {
+    try {
+        auto serialize_start_time = std::chrono::high_resolution_clock::now();
+        ov::save_model(model, save_path, compress_to_fp16);
+        auto serialize_finish_time = std::chrono::high_resolution_clock::now();
+        auto serialize_duration = std::chrono::duration_cast<std::chrono::milliseconds>(serialize_finish_time - serialize_start_time).count();
+        std::cout << "Save generated OpenVINO model to: " << save_path << " done. Time: " << serialize_duration << " ms\n";
+    }
+    catch (const ov::Exception& e) {
+        std::cerr << "[Warning] Exception during model serialization: " << e.what() << std::endl;
+    }
+}
+
+std::shared_ptr<ov::Model> create_from_gguf(const std::string& model_path, const ov::AnyMap& properties) {
     auto start_time = std::chrono::high_resolution_clock::now();
     std::cout << "Loading and unpacking model from: " << model_path << std::endl;
     auto [config, consts, qtypes] = load_gguf(model_path);
     auto load_finish_time = std::chrono::high_resolution_clock::now();
     std::cout << "Loading and unpacking model done. Time: " << std::chrono::duration_cast<std::chrono::milliseconds>(load_finish_time - start_time).count() << "ms" << std::endl;
-    std::cout << "Start generating OV model..." << std::endl;
-    
+    std::cout << "Start generating OpenVINO model..." << std::endl;
+
     std::shared_ptr<ov::Model> model;
 
     const std::string model_arch = std::get<std::string>(config.at("architecture"));
     if (!model_arch.compare("llama") || !model_arch.compare("qwen2")) {
         model = create_language_model(config, consts, qtypes);
+        if (properties.find(ov::cache_dir.name()) != properties.end()) {
+            std::string cache_dir = properties.at(ov::cache_dir.name()).as<std::string>();
+            if (!cache_dir.empty()) {
+                std::filesystem::path model_cache_dir(cache_dir);
+                std::filesystem::path gguf_model_path(model_path);
+                std::filesystem::path save_path = model_cache_dir / (gguf_model_path.stem().string() + "_openvino_model.xml");
+                save_openvino_model(model, save_path.string(), true);
+            }
+        }
     } else {
         OPENVINO_THROW("Unsupported model architecture '", model_arch, "'");
     }
     auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(
         std::chrono::high_resolution_clock::now() - load_finish_time).count();
     std::cout << "Model generation done. Time: " << duration << "ms" << std::endl;
+
     return model;
 }

src/cpp/src/gguf_utils/gguf_modeling.hpp

@@ -7,4 +7,4 @@
 
 #include "openvino/openvino.hpp"
 
-std::shared_ptr<ov::Model> create_from_gguf(const std::string& model_path);
+std::shared_ptr<ov::Model> create_from_gguf(const std::string& model_path, const ov::AnyMap& properties);

src/cpp/src/utils.cpp

@@ -292,10 +292,27 @@ bool is_gguf_model(const std::filesystem::path& file_path) {
 
 } // namespace
 
-std::shared_ptr<ov::Model> read_model(const std::filesystem::path& model_dir,  const ov::AnyMap& config) {
+std::shared_ptr<ov::Model> read_model(const std::filesystem::path& model_dir,  const ov::AnyMap& properties) {
     if (is_gguf_model(model_dir)) {
 #ifdef ENABLE_GGUF
-        return create_from_gguf(model_dir.string());
+        if (properties.find(ov::cache_dir.name()) != properties.end()) {
+            std::string cache_dir = properties.at(ov::cache_dir.name()).as<std::string>();
+            if (!cache_dir.empty()) {
+                std::filesystem::path model_cache_dir(cache_dir);
+                std::filesystem::path model_path = model_cache_dir / (model_dir.stem().string() + "_openvino_model.xml");
+                if (std::filesystem::exists(model_path)) {
+                    std::cout << "Found generated OpenVINO model: " << model_path.string() << ", skip creating from GGUF model.\n";
+                    auto start_time = std::chrono::high_resolution_clock::now();
+                    auto model = singleton_core().read_model(model_path, {}, properties);
+                    auto load_finish_time = std::chrono::high_resolution_clock::now();
+                    std::cout << "Loading OpenVINO model done. Time: " << std::chrono::duration_cast<std::chrono::milliseconds>(load_finish_time - start_time).count() << "ms" << std::endl;
+                    return model;
+                }
+            }
+            return create_from_gguf(model_dir.string(), properties);
+        } else {
+            return create_from_gguf(model_dir.string(), properties);
+        }
 #else
         OPENVINO_ASSERT("GGUF support is switched off. Please, recompile with 'cmake -DENABLE_GGUF=ON'");
 #endif
@@ -310,7 +327,7 @@ std::shared_ptr<ov::Model> read_model(const std::filesystem::path& model_dir,  c
             OPENVINO_THROW("Could not find a model in the directory '", model_dir, "'");
         }
 
-        return singleton_core().read_model(model_path, {}, config);
+        return singleton_core().read_model(model_path, {}, properties);
     }
 }