diff --git a/modelfile-repo/qwen3-14b-q4_k_m.Modelfile b/modelfile-repo/qwen3-14b-q4_k_m.Modelfile index 8bafa1e..4a180e6 100644 --- a/modelfile-repo/qwen3-14b-q4_k_m.Modelfile +++ b/modelfile-repo/qwen3-14b-q4_k_m.Modelfile @@ -20,12 +20,69 @@ PARAMETER presence_penalty 1.5 PARAMETER stop "<|im_start|>" PARAMETER stop "<|im_end|>" -# Template for Qwen3 chat format -TEMPLATE """{{ if .System }}<|im_start|>system -{{ .System }}<|im_end|> -{{ end }}<|im_start|>user -{{ .Prompt }}<|im_end|> -<|im_start|>assistant -""" +# Template for Qwen3 chat format with tool support +TEMPLATE """{{- $lastUserIdx := -1 -}} +{{- range $idx, $msg := .Messages -}} +{{- if eq $msg.Role "user" }}{{ $lastUserIdx = $idx }}{{ end -}} +{{- end }} +{{- if or .System .Tools }}<|im_start|>system +{{ if .System }} +{{ .System }} +{{- end }} +{{- if .Tools }} + +# Tools + +You may call one or more functions to assist with the user query. + +You are provided with function signatures within XML tags: + +{{- range .Tools }} +{"type": "function", "function": {{ .Function }}} +{{- end }} + + +For each function call, return a json object with function name and arguments within XML tags: + +{"name": , "arguments": } + +{{- end -}} +<|im_end|> +{{ end }} +{{- range $i, $_ := .Messages }} +{{- $last := eq (len (slice $.Messages $i)) 1 -}} +{{- if eq .Role "user" }}<|im_start|>user +{{ .Content }} +{{- if and $.IsThinkSet (eq $i $lastUserIdx) }} + {{- if $.Think -}} + {{- " "}}/think + {{- else -}} + {{- " "}}/no_think + {{- end -}} +{{- end }}<|im_end|> +{{ else if eq .Role "assistant" }}<|im_start|>assistant +{{ if (and $.IsThinkSet (and .Thinking (or $last (gt $i $lastUserIdx)))) -}} +{{ .Thinking }} +{{ end -}} +{{ if .Content }}{{ .Content }} +{{- else if .ToolCalls }} +{{ range .ToolCalls }}{"name": "{{ .Function.Name }}", "arguments": {{ .Function.Arguments }}} +{{ end }} +{{- end }}{{ if not $last }}<|im_end|> +{{ end }} +{{- else if eq .Role "tool" }}<|im_start|>user + +{{ .Content }} +<|im_end|> +{{ end }} +{{- if and (ne .Role "assistant") $last }}<|im_start|>assistant +{{ if and $.IsThinkSet (not $.Think) -}} + + + + +{{ end -}} +{{ end }} +{{- end }}""" SYSTEM """You are Qwen, created by Alibaba Cloud. You are a helpful assistant.""" diff --git a/modelfile-repo/qwen3-14b-q5_k_m.Modelfile b/modelfile-repo/qwen3-14b-q5_k_m.Modelfile index 0122ae4..e10bfc9 100644 --- a/modelfile-repo/qwen3-14b-q5_k_m.Modelfile +++ b/modelfile-repo/qwen3-14b-q5_k_m.Modelfile @@ -20,12 +20,69 @@ PARAMETER presence_penalty 1.5 PARAMETER stop "<|im_start|>" PARAMETER stop "<|im_end|>" -# Template for Qwen3 chat format -TEMPLATE """{{ if .System }}<|im_start|>system -{{ .System }}<|im_end|> -{{ end }}<|im_start|>user -{{ .Prompt }}<|im_end|> -<|im_start|>assistant -""" +# Template for Qwen3 chat format with tool support +TEMPLATE """{{- $lastUserIdx := -1 -}} +{{- range $idx, $msg := .Messages -}} +{{- if eq $msg.Role "user" }}{{ $lastUserIdx = $idx }}{{ end -}} +{{- end }} +{{- if or .System .Tools }}<|im_start|>system +{{ if .System }} +{{ .System }} +{{- end }} +{{- if .Tools }} + +# Tools + +You may call one or more functions to assist with the user query. + +You are provided with function signatures within XML tags: + +{{- range .Tools }} +{"type": "function", "function": {{ .Function }}} +{{- end }} + + +For each function call, return a json object with function name and arguments within XML tags: + +{"name": , "arguments": } + +{{- end -}} +<|im_end|> +{{ end }} +{{- range $i, $_ := .Messages }} +{{- $last := eq (len (slice $.Messages $i)) 1 -}} +{{- if eq .Role "user" }}<|im_start|>user +{{ .Content }} +{{- if and $.IsThinkSet (eq $i $lastUserIdx) }} + {{- if $.Think -}} + {{- " "}}/think + {{- else -}} + {{- " "}}/no_think + {{- end -}} +{{- end }}<|im_end|> +{{ else if eq .Role "assistant" }}<|im_start|>assistant +{{ if (and $.IsThinkSet (and .Thinking (or $last (gt $i $lastUserIdx)))) -}} +{{ .Thinking }} +{{ end -}} +{{ if .Content }}{{ .Content }} +{{- else if .ToolCalls }} +{{ range .ToolCalls }}{"name": "{{ .Function.Name }}", "arguments": {{ .Function.Arguments }}} +{{ end }} +{{- end }}{{ if not $last }}<|im_end|> +{{ end }} +{{- else if eq .Role "tool" }}<|im_start|>user + +{{ .Content }} +<|im_end|> +{{ end }} +{{- if and (ne .Role "assistant") $last }}<|im_start|>assistant +{{ if and $.IsThinkSet (not $.Think) -}} + + + + +{{ end -}} +{{ end }} +{{- end }}""" SYSTEM """You are Qwen, created by Alibaba Cloud. You are a helpful assistant.""" diff --git a/modelfile-repo/qwen3-14b-q8_0.Modelfile b/modelfile-repo/qwen3-14b-q8_0.Modelfile index f8e3ccf..78c1df5 100644 --- a/modelfile-repo/qwen3-14b-q8_0.Modelfile +++ b/modelfile-repo/qwen3-14b-q8_0.Modelfile @@ -5,7 +5,7 @@ FROM ./qwen3-14b-q8_0.gguf # Context and output parameters -PARAMETER num_ctx 40960 +PARAMETER num_ctx 8192 PARAMETER num_predict 32768 # Sampling parameters (non-thinking mode) @@ -20,12 +20,69 @@ PARAMETER presence_penalty 1.5 PARAMETER stop "<|im_start|>" PARAMETER stop "<|im_end|>" -# Template for Qwen3 chat format -TEMPLATE """{{ if .System }}<|im_start|>system -{{ .System }}<|im_end|> -{{ end }}<|im_start|>user -{{ .Prompt }}<|im_end|> -<|im_start|>assistant -""" +# Template for Qwen3 chat format with tool support +TEMPLATE """{{- $lastUserIdx := -1 -}} +{{- range $idx, $msg := .Messages -}} +{{- if eq $msg.Role "user" }}{{ $lastUserIdx = $idx }}{{ end -}} +{{- end }} +{{- if or .System .Tools }}<|im_start|>system +{{ if .System }} +{{ .System }} +{{- end }} +{{- if .Tools }} + +# Tools + +You may call one or more functions to assist with the user query. + +You are provided with function signatures within XML tags: + +{{- range .Tools }} +{"type": "function", "function": {{ .Function }}} +{{- end }} + + +For each function call, return a json object with function name and arguments within XML tags: + +{"name": , "arguments": } + +{{- end -}} +<|im_end|> +{{ end }} +{{- range $i, $_ := .Messages }} +{{- $last := eq (len (slice $.Messages $i)) 1 -}} +{{- if eq .Role "user" }}<|im_start|>user +{{ .Content }} +{{- if and $.IsThinkSet (eq $i $lastUserIdx) }} + {{- if $.Think -}} + {{- " "}}/think + {{- else -}} + {{- " "}}/no_think + {{- end -}} +{{- end }}<|im_end|> +{{ else if eq .Role "assistant" }}<|im_start|>assistant +{{ if (and $.IsThinkSet (and .Thinking (or $last (gt $i $lastUserIdx)))) -}} +{{ .Thinking }} +{{ end -}} +{{ if .Content }}{{ .Content }} +{{- else if .ToolCalls }} +{{ range .ToolCalls }}{"name": "{{ .Function.Name }}", "arguments": {{ .Function.Arguments }}} +{{ end }} +{{- end }}{{ if not $last }}<|im_end|> +{{ end }} +{{- else if eq .Role "tool" }}<|im_start|>user + +{{ .Content }} +<|im_end|> +{{ end }} +{{- if and (ne .Role "assistant") $last }}<|im_start|>assistant +{{ if and $.IsThinkSet (not $.Think) -}} + + + + +{{ end -}} +{{ end }} +{{- end }}""" SYSTEM """You are Qwen, created by Alibaba Cloud. You are a helpful assistant.""" diff --git a/modelfile-repo/qwen3-8b-q4_k_m.Modelfile b/modelfile-repo/qwen3-8b-q4_k_m.Modelfile index a2837aa..6c7278c 100644 --- a/modelfile-repo/qwen3-8b-q4_k_m.Modelfile +++ b/modelfile-repo/qwen3-8b-q4_k_m.Modelfile @@ -20,12 +20,69 @@ PARAMETER presence_penalty 1.5 PARAMETER stop "<|im_start|>" PARAMETER stop "<|im_end|>" -# Template for Qwen3 chat format -TEMPLATE """{{ if .System }}<|im_start|>system -{{ .System }}<|im_end|> -{{ end }}<|im_start|>user -{{ .Prompt }}<|im_end|> -<|im_start|>assistant -""" +# Template for Qwen3 chat format with tool support +TEMPLATE """{{- $lastUserIdx := -1 -}} +{{- range $idx, $msg := .Messages -}} +{{- if eq $msg.Role "user" }}{{ $lastUserIdx = $idx }}{{ end -}} +{{- end }} +{{- if or .System .Tools }}<|im_start|>system +{{ if .System }} +{{ .System }} +{{- end }} +{{- if .Tools }} + +# Tools + +You may call one or more functions to assist with the user query. + +You are provided with function signatures within XML tags: + +{{- range .Tools }} +{"type": "function", "function": {{ .Function }}} +{{- end }} + + +For each function call, return a json object with function name and arguments within XML tags: + +{"name": , "arguments": } + +{{- end -}} +<|im_end|> +{{ end }} +{{- range $i, $_ := .Messages }} +{{- $last := eq (len (slice $.Messages $i)) 1 -}} +{{- if eq .Role "user" }}<|im_start|>user +{{ .Content }} +{{- if and $.IsThinkSet (eq $i $lastUserIdx) }} + {{- if $.Think -}} + {{- " "}}/think + {{- else -}} + {{- " "}}/no_think + {{- end -}} +{{- end }}<|im_end|> +{{ else if eq .Role "assistant" }}<|im_start|>assistant +{{ if (and $.IsThinkSet (and .Thinking (or $last (gt $i $lastUserIdx)))) -}} +{{ .Thinking }} +{{ end -}} +{{ if .Content }}{{ .Content }} +{{- else if .ToolCalls }} +{{ range .ToolCalls }}{"name": "{{ .Function.Name }}", "arguments": {{ .Function.Arguments }}} +{{ end }} +{{- end }}{{ if not $last }}<|im_end|> +{{ end }} +{{- else if eq .Role "tool" }}<|im_start|>user + +{{ .Content }} +<|im_end|> +{{ end }} +{{- if and (ne .Role "assistant") $last }}<|im_start|>assistant +{{ if and $.IsThinkSet (not $.Think) -}} + + + + +{{ end -}} +{{ end }} +{{- end }}""" SYSTEM """You are Qwen, created by Alibaba Cloud. You are a helpful assistant.""" diff --git a/modelfile-repo/qwen3-8b-q5_k_m.Modelfile b/modelfile-repo/qwen3-8b-q5_k_m.Modelfile index 12fb4de..617622a 100644 --- a/modelfile-repo/qwen3-8b-q5_k_m.Modelfile +++ b/modelfile-repo/qwen3-8b-q5_k_m.Modelfile @@ -20,12 +20,69 @@ PARAMETER presence_penalty 1.5 PARAMETER stop "<|im_start|>" PARAMETER stop "<|im_end|>" -# Template for Qwen3 chat format -TEMPLATE """{{ if .System }}<|im_start|>system -{{ .System }}<|im_end|> -{{ end }}<|im_start|>user -{{ .Prompt }}<|im_end|> -<|im_start|>assistant -""" +# Template for Qwen3 chat format with tool support +TEMPLATE """{{- $lastUserIdx := -1 -}} +{{- range $idx, $msg := .Messages -}} +{{- if eq $msg.Role "user" }}{{ $lastUserIdx = $idx }}{{ end -}} +{{- end }} +{{- if or .System .Tools }}<|im_start|>system +{{ if .System }} +{{ .System }} +{{- end }} +{{- if .Tools }} + +# Tools + +You may call one or more functions to assist with the user query. + +You are provided with function signatures within XML tags: + +{{- range .Tools }} +{"type": "function", "function": {{ .Function }}} +{{- end }} + + +For each function call, return a json object with function name and arguments within XML tags: + +{"name": , "arguments": } + +{{- end -}} +<|im_end|> +{{ end }} +{{- range $i, $_ := .Messages }} +{{- $last := eq (len (slice $.Messages $i)) 1 -}} +{{- if eq .Role "user" }}<|im_start|>user +{{ .Content }} +{{- if and $.IsThinkSet (eq $i $lastUserIdx) }} + {{- if $.Think -}} + {{- " "}}/think + {{- else -}} + {{- " "}}/no_think + {{- end -}} +{{- end }}<|im_end|> +{{ else if eq .Role "assistant" }}<|im_start|>assistant +{{ if (and $.IsThinkSet (and .Thinking (or $last (gt $i $lastUserIdx)))) -}} +{{ .Thinking }} +{{ end -}} +{{ if .Content }}{{ .Content }} +{{- else if .ToolCalls }} +{{ range .ToolCalls }}{"name": "{{ .Function.Name }}", "arguments": {{ .Function.Arguments }}} +{{ end }} +{{- end }}{{ if not $last }}<|im_end|> +{{ end }} +{{- else if eq .Role "tool" }}<|im_start|>user + +{{ .Content }} +<|im_end|> +{{ end }} +{{- if and (ne .Role "assistant") $last }}<|im_start|>assistant +{{ if and $.IsThinkSet (not $.Think) -}} + + + + +{{ end -}} +{{ end }} +{{- end }}""" SYSTEM """You are Qwen, created by Alibaba Cloud. You are a helpful assistant.""" diff --git a/modelfile-repo/qwen3-8b-q8_0.Modelfile b/modelfile-repo/qwen3-8b-q8_0.Modelfile index 0f8dfca..a2f2a10 100644 --- a/modelfile-repo/qwen3-8b-q8_0.Modelfile +++ b/modelfile-repo/qwen3-8b-q8_0.Modelfile @@ -20,12 +20,69 @@ PARAMETER presence_penalty 1.5 PARAMETER stop "<|im_start|>" PARAMETER stop "<|im_end|>" -# Template for Qwen3 chat format -TEMPLATE """{{ if .System }}<|im_start|>system -{{ .System }}<|im_end|> -{{ end }}<|im_start|>user -{{ .Prompt }}<|im_end|> -<|im_start|>assistant -""" +# Template for Qwen3 chat format with tool support +TEMPLATE """{{- $lastUserIdx := -1 -}} +{{- range $idx, $msg := .Messages -}} +{{- if eq $msg.Role "user" }}{{ $lastUserIdx = $idx }}{{ end -}} +{{- end }} +{{- if or .System .Tools }}<|im_start|>system +{{ if .System }} +{{ .System }} +{{- end }} +{{- if .Tools }} + +# Tools + +You may call one or more functions to assist with the user query. + +You are provided with function signatures within XML tags: + +{{- range .Tools }} +{"type": "function", "function": {{ .Function }}} +{{- end }} + + +For each function call, return a json object with function name and arguments within XML tags: + +{"name": , "arguments": } + +{{- end -}} +<|im_end|> +{{ end }} +{{- range $i, $_ := .Messages }} +{{- $last := eq (len (slice $.Messages $i)) 1 -}} +{{- if eq .Role "user" }}<|im_start|>user +{{ .Content }} +{{- if and $.IsThinkSet (eq $i $lastUserIdx) }} + {{- if $.Think -}} + {{- " "}}/think + {{- else -}} + {{- " "}}/no_think + {{- end -}} +{{- end }}<|im_end|> +{{ else if eq .Role "assistant" }}<|im_start|>assistant +{{ if (and $.IsThinkSet (and .Thinking (or $last (gt $i $lastUserIdx)))) -}} +{{ .Thinking }} +{{ end -}} +{{ if .Content }}{{ .Content }} +{{- else if .ToolCalls }} +{{ range .ToolCalls }}{"name": "{{ .Function.Name }}", "arguments": {{ .Function.Arguments }}} +{{ end }} +{{- end }}{{ if not $last }}<|im_end|> +{{ end }} +{{- else if eq .Role "tool" }}<|im_start|>user + +{{ .Content }} +<|im_end|> +{{ end }} +{{- if and (ne .Role "assistant") $last }}<|im_start|>assistant +{{ if and $.IsThinkSet (not $.Think) -}} + + + + +{{ end -}} +{{ end }} +{{- end }}""" SYSTEM """You are Qwen, created by Alibaba Cloud. You are a helpful assistant.""" diff --git a/scripts/hf-llm-install.py b/scripts/hf-llm-install.py index aac7c7c..ff86aee 100755 --- a/scripts/hf-llm-install.py +++ b/scripts/hf-llm-install.py @@ -355,9 +355,9 @@ def create_ollama_model(modelfile_path, gguf_path, model_name, capabilities=None ) if result.returncode == 0: - print(f"✓ Model '{model_name}' created successfully") + # Success - output will be shown by the caller if result.stdout: - print(f" {result.stdout.strip()}") + print(result.stdout.strip()) else: print(f"✗ Failed to create model") if result.stderr: