From ccbe95ac1ee7cf5b88bbab5c1c8ffe6e2ce2da3aec0ce52c57f996638de124ee Mon Sep 17 00:00:00 2001 From: SERVICE GPGPU Date: Mon, 19 Jan 2026 21:55:49 +0000 Subject: [PATCH] additional coding-specific modelfiles --- ...ek-coder-v2-lite-instruct-q4_k_m.Modelfile | 48 +++++++++++ ...qwen2.5-coder-1.5b-instruct-q8_0.Modelfile | 80 +++++++++++++++++++ ...wen2.5-coder-14b-instruct-q5_k_m.Modelfile | 80 +++++++++++++++++++ 3 files changed, 208 insertions(+) create mode 100644 modelfile-repo/deepseek-coder-v2-lite-instruct-q4_k_m.Modelfile create mode 100644 modelfile-repo/qwen2.5-coder-1.5b-instruct-q8_0.Modelfile create mode 100644 modelfile-repo/qwen2.5-coder-14b-instruct-q5_k_m.Modelfile diff --git a/modelfile-repo/deepseek-coder-v2-lite-instruct-q4_k_m.Modelfile b/modelfile-repo/deepseek-coder-v2-lite-instruct-q4_k_m.Modelfile new file mode 100644 index 0000000..16cc668 --- /dev/null +++ b/modelfile-repo/deepseek-coder-v2-lite-instruct-q4_k_m.Modelfile @@ -0,0 +1,48 @@ +# Modelfile for deepseek-coder-v2-lite-instruct-q4_k_m +# hf_upstream: https://huggingface.co/bartowski/DeepSeek-Coder-V2-Lite-Instruct-GGUF/resolve/main/DeepSeek-Coder-V2-Lite-Instruct-Q4_K_M.gguf +# quantization: Q4_K_M +# capabilities: tools +# sha256: + +FROM ./DeepSeek-Coder-V2-Lite-Instruct-Q4_K_M.gguf + +# System prompt - customize for your use case +SYSTEM """You are Deepseek-Coder-V2-Lite, A specialized Large Language model for coding, logic and math.""" + +# Parameters - refer to manufacturer's recommendations +# https://huggingface.co/bartowski/DeepSeek-Coder-V2-Lite-Instruct-GGUF +PARAMETER temperature 0.2 +PARAMETER top_p 0.9 + +PARAMETER num_ctx 18432 +PARAMETER num_predict 32768 +PARAMETER repeat_penalty 1.0 +PARAMETER stop "<|im_start|>" +PARAMETER stop "<|im_end|>" +PARAMETER stop "" + +# Template - adjust based on model's chat template +TEMPLATE """ +{{- if .Suffix }}<｜fim▁begin｜>{{ .Prompt }}<｜fim▁hole｜>{{ .Suffix }}<｜fim▁end｜> +{{- else if .Messages }}<｜begin▁of▁sentence｜> +{{- $system := "" }} +{{- range $i, $_ := .Messages }} +{{- if eq .Role "system" }} +{{- $system = printf "%s %s" $system .Content }} +{{- else if eq .Role "user" }} +{{- if $system }}{{ $system }} +{{ $system = "" }} +{{ end }}User: {{ .Content }} + +{{ if eq (len (slice $.Messages $i)) 1 }}Assistant: +{{- end }} +{{- else if eq .Role "assistant" }}Assistant: {{ .Content }}<｜end▁of▁sentence｜> +{{- end }} +{{- end }} +{{- else }} +{{- if .System }}{{ .System }} +{{- end }} +{{- if .Prompt }}User: {{ .Prompt }} +{{- end }}Assistant:{{ .Response }} +{{- end }} +""" \ No newline at end of file diff --git a/modelfile-repo/qwen2.5-coder-1.5b-instruct-q8_0.Modelfile b/modelfile-repo/qwen2.5-coder-1.5b-instruct-q8_0.Modelfile new file mode 100644 index 0000000..46db7c6 --- /dev/null +++ b/modelfile-repo/qwen2.5-coder-1.5b-instruct-q8_0.Modelfile @@ -0,0 +1,80 @@ +# Modelfile for qwen2.5-coder:1.5b-instruct-q8_0 +# hf_upstream: https://huggingface.co/Qwen/Qwen2.5-Coder-1.5B-Instruct-GGUF/resolve/main/qwen2.5-coder-1.5b-instruct-q8_0.gguf +# quantization: Q8_0 +# capabilities: tools,thinking +# sha256: + +FROM ./qwen2.5-coder-1.5b-instruct-q8_0.gguf + +# System prompt - customize for your use case +SYSTEM """You are Qwen, created by Alibaba Cloud. You are a helpful assistant.""" + +# Parameters - refer to manufacturer's recommendations +# https://huggingface.co/Qwen/Qwen2.5-Coder-1.5B-Instruct-GGUF +PARAMETER temperature 0.7 +PARAMETER top_p 0.8 +PARAMETER top_k 20 +PARAMETER min_p 0.00 +PARAMETER repeat_penalty 1.0 +PARAMETER presence_penalty 1.5 +PARAMETER num_ctx 32768 +PARAMETER num_predict 32768 + +PARAMETER stop "<|im_end|>" +PARAMETER stop "<|end|>" +PARAMETER stop "" + +# Template - adjust based on model's chat template +TEMPLATE """ +{{- if .Suffix }}<|fim_prefix|>{{ .Prompt }}<|fim_suffix|>{{ .Suffix }}<|fim_middle|> +{{- else if .Messages }} +{{- if or .System .Tools }}<|im_start|>system +{{- if .System }} +{{ .System }} +{{- end }} +{{- if .Tools }} + +# Tools + +You may call one or more functions to assist with the user query. + +You are provided with function signatures within : + +{{- range .Tools }} +{"type": "function", "function": {{ .Function }}} +{{- end }} + + +For each function call, return a json object with function name and arguments within with NO other text. Do not include any backticks or ```json. + +{"name": , "arguments": } + +{{- end }}<|im_end|> +{{ end }} +{{- range $i, $_ := .Messages }} +{{- $last := eq (len (slice $.Messages $i)) 1 -}} +{{- if eq .Role "user" }}<|im_start|>user +{{ .Content }}<|im_end|> +{{ else if eq .Role "assistant" }}<|im_start|>assistant +{{ if .Content }}{{ .Content }} +{{- else if .ToolCalls }} +{{ range .ToolCalls }}{"name": "{{ .Function.Name }}", "arguments": {{ .Function.Arguments }}} +{{ end }} +{{- end }}{{ if not $last }}<|im_end|> +{{ end }} +{{- else if eq .Role "tool" }}<|im_start|>user + +{{ .Content }} +<|im_end|> +{{ end }} +{{- if and (ne .Role "assistant") $last }}<|im_start|>assistant +{{ end }} +{{- end }} +{{- else }} +{{- if .System }}<|im_start|>system +{{ .System }}<|im_end|> +{{ end }}{{ if .Prompt }}<|im_start|>user +{{ .Prompt }}<|im_end|> +{{ end }}<|im_start|>assistant +{{ end }}{{ .Response }}{{ if .Response }}<|im_end|>{{ end }} +""" \ No newline at end of file diff --git a/modelfile-repo/qwen2.5-coder-14b-instruct-q5_k_m.Modelfile b/modelfile-repo/qwen2.5-coder-14b-instruct-q5_k_m.Modelfile new file mode 100644 index 0000000..c48a58f --- /dev/null +++ b/modelfile-repo/qwen2.5-coder-14b-instruct-q5_k_m.Modelfile @@ -0,0 +1,80 @@ +# Modelfile for qwen2.5-coder:14b-instruct-q5_k_m +# hf_upstream: https://huggingface.co/Qwen/Qwen2.5-Coder-14B-Instruct-GGUF/resolve/main/qwen2.5-coder-14b-instruct-q5_k_m.gguf +# quantization: Q5_K_M +# capabilities: tools,thinking +# sha256: + +FROM ./qwen2.5-coder-14b-instruct-q5_k_m.gguf + +# System prompt - customize for your use case +SYSTEM """You are Qwen, created by Alibaba Cloud. You are a helpful assistant.""" + +# Parameters - refer to manufacturer's recommendations +# https://huggingface.co/Qwen/Qwen2.5-Coder-14B-Instruct-GGUF +PARAMETER temperature 0.7 +PARAMETER top_p 0.8 +PARAMETER top_k 20 +PARAMETER min_p 0.00 +PARAMETER repeat_penalty 1.0 +PARAMETER presence_penalty 1.5 +PARAMETER num_ctx 32768 +PARAMETER num_predict 32768 + +PARAMETER stop "<|im_end|>" +PARAMETER stop "<|end|>" +PARAMETER stop "" + +# Template - adjust based on model's chat template +TEMPLATE """ +{{- if .Suffix }}<|fim_prefix|>{{ .Prompt }}<|fim_suffix|>{{ .Suffix }}<|fim_middle|> +{{- else if .Messages }} +{{- if or .System .Tools }}<|im_start|>system +{{- if .System }} +{{ .System }} +{{- end }} +{{- if .Tools }} + +# Tools + +You may call one or more functions to assist with the user query. + +You are provided with function signatures within : + +{{- range .Tools }} +{"type": "function", "function": {{ .Function }}} +{{- end }} + + +For each function call, return a json object with function name and arguments within with NO other text. Do not include any backticks or ```json. + +{"name": , "arguments": } + +{{- end }}<|im_end|> +{{ end }} +{{- range $i, $_ := .Messages }} +{{- $last := eq (len (slice $.Messages $i)) 1 -}} +{{- if eq .Role "user" }}<|im_start|>user +{{ .Content }}<|im_end|> +{{ else if eq .Role "assistant" }}<|im_start|>assistant +{{ if .Content }}{{ .Content }} +{{- else if .ToolCalls }} +{{ range .ToolCalls }}{"name": "{{ .Function.Name }}", "arguments": {{ .Function.Arguments }}} +{{ end }} +{{- end }}{{ if not $last }}<|im_end|> +{{ end }} +{{- else if eq .Role "tool" }}<|im_start|>user + +{{ .Content }} +<|im_end|> +{{ end }} +{{- if and (ne .Role "assistant") $last }}<|im_start|>assistant +{{ end }} +{{- end }} +{{- else }} +{{- if .System }}<|im_start|>system +{{ .System }}<|im_end|> +{{ end }}{{ if .Prompt }}<|im_start|>user +{{ .Prompt }}<|im_end|> +{{ end }}<|im_start|>assistant +{{ end }}{{ .Response }}{{ if .Response }}<|im_end|>{{ end }} +""" \ No newline at end of file