Repository navigation
Expand file tree
/
Copy pathmodel_utils.py
More file actions
345 lines (292 loc) · 13.4 KB
/
Copy pathmodel_utils.py
File metadata and controls
345 lines (292 loc) · 13.4 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
"""
Utility functions for managing local LLM models
Used for downloading, verifying, and testing local models
"""
import os
import json
import shutil
import hashlib
from pathlib import Path
from typing import Dict, Any, Optional, List
import requests
from tqdm import tqdm
import streamlit as st
# Default models available for download
DEFAULT_MODELS = {
"phi-2.Q4_K_M.gguf": {
"name": "Phi-2 (Microsoft)",
"size": "2.9GB",
"description": "Compact but powerful model from Microsoft",
"url": "https://huggingface.co/TheBloke/phi-2-GGUF/resolve/main/phi-2.Q4_K_M.gguf",
"md5": "bc33a1791219b5abb9432f3f581a55f3",
"recommended_ctx": 4096,
"category": "Small",
"quantization": "Q4_K_M",
"family": "phi"
},
"mistral-7b-instruct-v0.2.Q4_K_M.gguf": {
"name": "Mistral 7B Instruct",
"size": "4.37GB",
"description": "Efficient model with strong instruction following",
"url": "https://huggingface.co/TheBloke/Mistral-7B-Instruct-v0.2-GGUF/resolve/main/mistral-7b-instruct-v0.2.Q4_K_M.gguf",
"md5": "71512e9f27482fa9a00039c9b452b94e",
"recommended_ctx": 8192,
"category": "Medium",
"quantization": "Q4_K_M",
"family": "mistral"
},
"llama-2-7b-chat.Q4_K_M.gguf": {
"name": "Llama 2 7B Chat",
"size": "4.78GB",
"description": "Meta's Llama 2 optimized for chat",
"url": "https://huggingface.co/TheBloke/Llama-2-7B-Chat-GGUF/resolve/main/llama-2-7b-chat.Q4_K_M.gguf",
"md5": "84a0925f5f6073238f64684327587e9d",
"recommended_ctx": 4096,
"category": "Medium",
"quantization": "Q4_K_M",
"family": "llama"
},
"tinyllama-1.1b-chat-v1.0.Q4_K_M.gguf": {
"name": "TinyLlama 1.1B Chat",
"size": "610MB",
"description": "Ultra-compact model, great for testing",
"url": "https://huggingface.co/TheBloke/TinyLlama-1.1B-Chat-v1.0-GGUF/resolve/main/tinyllama-1.1b-chat-v1.0.Q4_K_M.gguf",
"md5": "3951db6b4de4d4af8833f50231d8a8aa",
"recommended_ctx": 2048,
"category": "Tiny",
"quantization": "Q4_K_M",
"family": "llama"
}
}
def compute_md5(file_path: str, chunk_size: int = 1024 * 1024) -> str:
"""Compute a file MD5 without loading the whole file into memory."""
digest = hashlib.md5()
with open(file_path, 'rb') as f:
for chunk in iter(lambda: f.read(chunk_size), b""):
digest.update(chunk)
return digest.hexdigest()
def ensure_models_directory() -> str:
"""
Ensure the models directory exists and return its path
Returns:
Path to models directory
"""
models_dir = os.path.join(os.getcwd(), "models")
os.makedirs(models_dir, exist_ok=True)
return models_dir
def get_model_info(model_filename: str) -> Optional[Dict[str, Any]]:
"""
Get information about a model from the DEFAULT_MODELS dictionary
Args:
model_filename: Filename of the model
Returns:
Model information dictionary or None if not found
"""
return DEFAULT_MODELS.get(model_filename)
def download_model(model_filename: str, force: bool = False) -> Optional[str]:
"""
Download a model from the default models list
Args:
model_filename: Filename of the model to download
force: Whether to force download even if the file exists
Returns:
Path to the downloaded model file, or None if download failed
"""
# Get model info
model_info = get_model_info(model_filename)
if not model_info:
st.error(f"Model {model_filename} not found in available models")
return None
# Prepare path
models_dir = ensure_models_directory()
model_path = os.path.join(models_dir, model_filename)
# Check if already exists and not forcing redownload
if os.path.exists(model_path) and not force:
# Verify integrity
st.info(f"Model already exists at {model_path}. Verifying integrity...")
file_hash = compute_md5(model_path)
if file_hash == model_info.get('md5'):
st.success(f"Model integrity verified. Ready to use.")
return model_path
else:
st.warning(f"Model integrity check failed. Will redownload.")
# Download model
url = model_info.get('url')
if not url:
st.error("Model URL not found")
return None
st.info(f"Downloading {model_info.get('name')} ({model_info.get('size')})...")
st.info(f"This may take some time depending on your internet connection.")
try:
# Stream download with progress bar
temp_path = model_path + ".download"
response = requests.get(url, stream=True, timeout=(10, 120))
response.raise_for_status()
total_size = int(response.headers.get('content-length', 0))
with open(temp_path, 'wb') as f:
with tqdm(total=total_size, unit='B', unit_scale=True, desc=model_filename) as pbar:
for chunk in response.iter_content(chunk_size=8192):
if chunk:
f.write(chunk)
pbar.update(len(chunk))
# Verify download
file_hash = compute_md5(temp_path)
if file_hash == model_info.get('md5'):
# Move to final location
shutil.move(temp_path, model_path)
st.success(f"Download complete and verified. Model saved to {model_path}")
return model_path
else:
os.remove(temp_path)
st.error("Downloaded file integrity check failed. Please try again.")
return None
except Exception as e:
st.error(f"Error downloading model: {str(e)}")
# Clean up partial download
if 'temp_path' in locals() and os.path.exists(temp_path):
os.remove(temp_path)
return None
def list_available_models() -> Dict[str, Dict[str, Any]]:
"""
List available models and their download status
Returns:
Dictionary with model information and download status
"""
models_dir = ensure_models_directory()
result = {}
# Check default models
for filename, info in DEFAULT_MODELS.items():
model_path = os.path.join(models_dir, filename)
is_downloaded = os.path.exists(model_path)
# Verify hash if downloaded
verified = False
if is_downloaded:
try:
file_hash = compute_md5(model_path)
verified = file_hash == info.get('md5')
except Exception:
verified = False
result[filename] = {
**info,
"downloaded": is_downloaded,
"verified": verified,
"path": model_path if is_downloaded else None
}
# Add custom models (those in models directory not in DEFAULT_MODELS)
for filename in os.listdir(models_dir):
if filename.endswith('.gguf') and filename not in DEFAULT_MODELS:
model_path = os.path.join(models_dir, filename)
file_size = os.path.getsize(model_path)
size_mb = file_size / (1024 * 1024)
size_str = f"{size_mb:.2f}MB" if size_mb < 1024 else f"{size_mb/1024:.2f}GB"
result[filename] = {
"name": filename,
"size": size_str,
"description": "Custom model",
"downloaded": True,
"verified": True, # We trust user's custom models
"path": model_path,
"category": "Custom",
"recommended_ctx": 4096 # Default value
}
return result
def show_model_download_ui() -> Optional[str]:
"""
Display a Streamlit UI for downloading and managing models
Returns:
Path to the selected model or None if no model selected
"""
st.header("Local LLM Model Manager")
# Get available models
models = list_available_models()
# Tab UI for model operations
tab1, tab2 = st.tabs(["Download Models", "Manage Models"])
selected_model_path = None
with tab1:
st.subheader("Download Pre-configured Models")
# Group models by category
models_by_category = {}
for filename, info in models.items():
category = info.get('category', 'Other')
if category not in models_by_category:
models_by_category[category] = []
models_by_category[category].append((filename, info))
# Display models by category
for category in sorted(models_by_category.keys()):
with st.expander(f"{category} Models", expanded=True):
for filename, info in sorted(models_by_category[category]):
col1, col2, col3 = st.columns([3, 1, 1])
with col1:
st.write(f"**{info['name']}** ({info['size']})")
st.write(info['description'])
with col2:
status = "✅" if info['downloaded'] and info['verified'] else \
"⚠️" if info['downloaded'] and not info['verified'] else "❌"
st.write(f"Status: {status}")
with col3:
if not info['downloaded'] or not info['verified']:
if st.button("Download", key=f"download_{filename}"):
path = download_model(filename)
# Force refresh of the page
st.rerun()
else:
if st.button("Redownload", key=f"redownload_{filename}"):
path = download_model(filename, force=True)
# Force refresh of the page
st.rerun()
# Upload custom model
st.subheader("Upload Custom Model")
uploaded_file = st.file_uploader("Upload a GGUF model", type=['gguf'])
if uploaded_file is not None:
# ``st.file_uploader`` keeps the file across reruns, so remember which upload
# was already written; otherwise the save + rerun below loops forever.
upload_token = f"{getattr(uploaded_file, 'file_id', '')}:{uploaded_file.name}:{uploaded_file.size}"
if st.session_state.get("last_processed_model_upload") == upload_token:
st.info(f"'{Path(uploaded_file.name).name}' has been saved. Clear the uploader to upload another model.")
else:
# Save the uploaded file
models_dir = ensure_models_directory()
safe_name = Path(uploaded_file.name).name
if not safe_name.lower().endswith(".gguf"):
st.error("Only .gguf model files are supported.")
return selected_model_path
model_path = os.path.join(models_dir, safe_name)
with open(model_path, "wb") as f:
f.write(uploaded_file.getbuffer())
st.session_state["last_processed_model_upload"] = upload_token
st.success(f"Model uploaded successfully to {model_path}")
# Force refresh of the page
st.rerun()
with tab2:
st.subheader("Manage Downloaded Models")
downloaded_models = {k: v for k, v in models.items() if v['downloaded']}
if not downloaded_models:
st.info("No models downloaded yet. Use the Download Models tab to get started.")
else:
# Display downloaded models with management options
for filename, info in downloaded_models.items():
with st.expander(info['name'], expanded=False):
col1, col2 = st.columns([3, 1])
with col1:
st.write(f"**Path:** {info['path']}")
st.write(f"**Size:** {info['size']}")
st.write(f"**Category:** {info.get('category', 'Custom')}")
if 'family' in info:
st.write(f"**Family:** {info['family']}")
if 'quantization' in info:
st.write(f"**Quantization:** {info['quantization']}")
st.write(f"**Recommended Context Size:** {info.get('recommended_ctx', 'Unknown')}")
with col2:
if st.button("Delete", key=f"delete_{filename}"):
try:
os.remove(info['path'])
st.success(f"Deleted model: {filename}")
# Force refresh of the page
st.rerun()
except Exception as e:
st.error(f"Error deleting model: {str(e)}")
if st.button("Select", key=f"select_{filename}"):
selected_model_path = info['path']
# Highlight selection
st.success(f"Selected model: {filename}")
return selected_model_path