mmproj upgrade

web_app.py

@@ -288,7 +288,8 @@ def run_install_job(job_id: str, modelfile_path: str):
                 install_jobs[job_id]['error'] = str(e)
 
 
-def run_huggingface_install_job(job_id: str, model_name: str, modelfile_content: str, file_url: str, gguf_filename: str):
+def run_huggingface_install_job(job_id: str, model_name: str, modelfile_content: str, file_url: str, gguf_filename: str, 
+                                mmproj_url: str = None, mmproj_filename: str = None):
     """Run HuggingFace model installation in background thread."""
     with install_lock:
         install_jobs[job_id]['status'] = 'running'
@@ -305,6 +306,7 @@ def run_huggingface_install_job(job_id: str, model_name: str, modelfile_content:
             return install_jobs[job_id].get('cancelled', False)
     
     temp_gguf = None
+    temp_mmproj = None
     temp_modelfile = None
     
     try:
@@ -314,16 +316,27 @@ def run_huggingface_install_job(job_id: str, model_name: str, modelfile_content:
         temp_gguf.close()
         gguf_path = temp_gguf.name
         
+        mmproj_path = None
+        if mmproj_url and mmproj_filename:
+            temp_mmproj = tempfile.NamedTemporaryFile(suffix='.gguf', delete=False)
+            temp_mmproj.close()
+            mmproj_path = temp_mmproj.name
+        
         temp_modelfile = tempfile.NamedTemporaryFile(mode='w', suffix='.Modelfile', delete=False)
         temp_modelfile.write(modelfile_content)
         temp_modelfile.close()
         modelfile_path = temp_modelfile.name
         
-        # Use existing download_file function with callbacks
+        # Download main GGUF file
         hf_install_module.download_file(file_url, gguf_path, gguf_filename, should_cancel, update_progress)
         
-        # Use existing create_ollama_model function
-        hf_install_module.create_ollama_model(modelfile_path, gguf_path, model_name)
+        # Download mmproj file if specified
+        if mmproj_path and mmproj_url:
+            update_progress('Downloading mmproj file for vision support...')
+            hf_install_module.download_file(mmproj_url, mmproj_path, mmproj_filename, should_cancel, update_progress)
+        
+        # Create Ollama model with both files
+        hf_install_module.create_ollama_model(modelfile_path, gguf_path, model_name, mmproj_path=mmproj_path)
         
         # Save Modelfile to repo
         normalized_name = model_name.replace(':', '-')
@@ -353,6 +366,8 @@ def run_huggingface_install_job(job_id: str, model_name: str, modelfile_content:
         # Clean up temp files
         if temp_gguf and os.path.exists(temp_gguf.name):
             os.unlink(temp_gguf.name)
+        if temp_mmproj and os.path.exists(temp_mmproj.name):
+            os.unlink(temp_mmproj.name)
         if temp_modelfile and os.path.exists(temp_modelfile.name):
             os.unlink(temp_modelfile.name)
 
@@ -707,11 +722,31 @@ def generate_modelfile_response(org: str, repo: str, gguf_filename: str, file_ur
         quant_match = re.search(r'[._-](Q[0-9]+_[KLM0-9]+(?:_[LSM])?)', gguf_filename, re.IGNORECASE)
         quantization = quant_match.group(1).upper() if quant_match else 'unspecified'
         
+        # Detect if model might support vision (multimodal models)
+        # Common patterns: ministral-3, qwen-vl, llava, etc.
+        is_multimodal = any(pattern in repo.lower() for pattern in 
+            ['ministral-3', 'qwen-vl', 'qwen2-vl', 'qwen3-vl', 'llava', 'minicpm-v', 'phi-3-vision'])
+        
+        # Build capabilities list
+        capabilities = ['tools']  # Most modern models support tools
+        if is_multimodal:
+            capabilities.append('vision')
+        
+        # Build mmproj config if multimodal
+        mmproj_config = ''
+        if is_multimodal:
+            # Try to use unsloth for mmproj (usually has more options)
+            mmproj_org = 'unsloth' if 'ministral' in repo.lower() or 'qwen' in repo.lower() else org
+            mmproj_config = f"""# 
+# mmproj_url: https://huggingface.co/{mmproj_org}/{repo}
+# mmproj_quant: BF16
+"""
+        
         # Create Modelfile skeleton with relative path (like CLI does)
         modelfile_content = f"""# Modelfile for {full_name}
 # hf_upstream: {file_url}
 # quantization: {quantization}
-# capabilities: tools
+# capabilities: {', '.join(capabilities)}{mmproj_config}
 # sha256: <add_sha256_checksum_here>
 
 FROM ./{gguf_filename}
@@ -764,6 +799,8 @@ def api_create_from_modelfile():
     modelfile_content = data.get('modelfile_content', '')
     file_url = data.get('file_url', '')
     gguf_filename = data.get('gguf_filename', '')
+    mmproj_url = data.get('mmproj_url', '').strip() or None
+    mmproj_filename = data.get('mmproj_filename', '').strip() or None
     
     if not model_name or not modelfile_content or not file_url:
         return jsonify({'error': 'Missing required parameters'}), 400
@@ -785,7 +822,7 @@ def api_create_from_modelfile():
         # Start background thread
         thread = threading.Thread(
             target=run_huggingface_install_job,
-            args=(job_id, model_name, modelfile_content, file_url, gguf_filename)
+            args=(job_id, model_name, modelfile_content, file_url, gguf_filename, mmproj_url, mmproj_filename)
         )
         thread.daemon = True
         thread.start()
@@ -795,11 +832,6 @@ def api_create_from_modelfile():
             'job_id': job_id,
             'message': 'Installation started'
         })
-        
-    except Exception as e:
-        return jsonify({'error': str(e)}), 500
-
-
 @app.route('/api/install/modelfile', methods=['POST'])
 def api_install_from_modelfile():
     """Start installation of a model from an existing Modelfile as background job."""