Skip to content

Commit 46d9caa

Browse files
authored
model-conversion : add mmproj conversion target (#15628)
This commit adds a new target to the Makefile for converting models that are multimodal. This target will convert the original model and in addition also create the mmproj GGUF model. The motivation for this change is that for models that are multimodal, for example those that contain a vision encoders, we will often want to upload both the quantized model and the vision encoder model to HuggingFace. Example usage: ```console $ make causal-convert-mm-model MODEL_PATH=~/work/ai/models/gemma-3-4b-it-qat-q4_0-unquantized/ ... The environment variable CONVERTED_MODEL can be set to this path using: export CONVERTED_MODEL=/home/danbev/work/ai/llama.cpp/models/gemma-3-4b-it-qat-q4_0-unquantized.gguf The mmproj model was created in /home/danbev/work/ai/llama.cpp/models/mmproj-gemma-3-4b-it-qat-q4_0-unquantized.gguf ``` The converted original model can then be quantized, and after that both the quantized model and the mmproj file can then be uploaded to HuggingFace. Refs: https://huggingface.co/ggml-org/gemma-3-4b-it-qat-GGUF/tree/main
1 parent 5a0e3ef commit 46d9caa

File tree

2 files changed

+43
-5
lines changed

2 files changed

+43
-5
lines changed

examples/model-conversion/Makefile

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,20 @@ causal-convert-model:
3737
METADATA_OVERRIDE="$(METADATA_OVERRIDE)" \
3838
./scripts/causal/convert-model.sh
3939

40+
causal-convert-mm-model-bf16: OUTTYPE=bf16
41+
causal-convert-mm-model-bf16: MM_OUTTYPE=f16
42+
causal-convert-mm-model-bf16: causal-convert-mm-model
43+
44+
causal-convert-mm-model:
45+
$(call validate_model_path,causal-convert-mm-model)
46+
@MODEL_NAME="$(MODEL_NAME)" OUTTYPE="$(OUTTYPE)" MODEL_PATH="$(MODEL_PATH)" \
47+
METADATA_OVERRIDE="$(METADATA_OVERRIDE)" \
48+
./scripts/causal/convert-model.sh
49+
50+
@MODEL_NAME="$(MODEL_NAME)" OUTTYPE="$(MM_OUTTYPE)" MODEL_PATH="$(MODEL_PATH)" \
51+
METADATA_OVERRIDE="$(METADATA_OVERRIDE)" \
52+
./scripts/causal/convert-model.sh --mmproj
53+
4054
causal-run-original-model:
4155
$(call validate_model_path,causal-run-original-model)
4256
@MODEL_PATH="$(MODEL_PATH)" ./scripts/causal/run-org-model.py
Lines changed: 29 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,21 @@
11
#!/bin/bash
22

3+
set -e
4+
5+
# Parse command line arguments
6+
MMPROJ=""
7+
while [[ $# -gt 0 ]]; do
8+
case $1 in
9+
--mmproj)
10+
MMPROJ="--mmproj"
11+
shift
12+
;;
13+
*)
14+
shift
15+
;;
16+
esac
17+
done
18+
319
MODEL_NAME="${MODEL_NAME:-$(basename "$MODEL_PATH")}"
420
OUTPUT_DIR="${OUTPUT_DIR:-../../models}"
521
TYPE="${OUTTYPE:-f16}"
@@ -11,12 +27,20 @@ echo "Model name: ${MODEL_NAME}"
1127
echo "Data type: ${TYPE}"
1228
echo "Converted model path:: ${CONVERTED_MODEL}"
1329
echo "Metadata override: ${METADATA_OVERRIDE}"
14-
python ../../convert_hf_to_gguf.py --verbose \
15-
${MODEL_PATH} \
16-
--outfile ${CONVERTED_MODEL} \
17-
--outtype ${TYPE} \
18-
--metadata "${METADATA_OVERRIDE}"
30+
31+
CMD_ARGS=("python" "../../convert_hf_to_gguf.py" "--verbose")
32+
CMD_ARGS+=("${MODEL_PATH}")
33+
CMD_ARGS+=("--outfile" "${CONVERTED_MODEL}")
34+
CMD_ARGS+=("--outtype" "${TYPE}")
35+
[[ -n "$METADATA_OVERRIDE" ]] && CMD_ARGS+=("--metadata" "${METADATA_OVERRIDE}")
36+
[[ -n "$MMPROJ" ]] && CMD_ARGS+=("${MMPROJ}")
37+
38+
"${CMD_ARGS[@]}"
1939

2040
echo ""
2141
echo "The environment variable CONVERTED_MODEL can be set to this path using:"
2242
echo "export CONVERTED_MODEL=$(realpath ${CONVERTED_MODEL})"
43+
if [[ -n "$MMPROJ" ]]; then
44+
mmproj_file="${OUTPUT_DIR}/mmproj-$(basename "${CONVERTED_MODEL}")"
45+
echo "The mmproj model was created in $(realpath "$mmproj_file")"
46+
fi

0 commit comments

Comments
 (0)