Repository navigation
Expand file tree
/
Copy pathutils.py
More file actions
505 lines (425 loc) · 19.3 KB
/
Copy pathutils.py
File metadata and controls
505 lines (425 loc) · 19.3 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
import os
import uuid
import tempfile
import mimetypes
from datetime import datetime
from typing import Optional, Tuple, Dict, Any, List
import streamlit as st
from database import get_session, session_scope
from models import Conversation, Message, File, User, Settings
from sqlalchemy.sql import func
# Import MS DLP functionality for file sensitivity checking
# This import is done in a try-except to allow the app to work without MS DLP integration
try:
from ms_dlp import scan_file_for_sensitivity, is_dlp_integration_enabled
MS_DLP_AVAILABLE = True
except ImportError:
MS_DLP_AVAILABLE = False
def generate_unique_id() -> str:
"""Generate a unique ID for files or conversations"""
return str(uuid.uuid4())
def save_uploaded_file(uploaded_file, content_override: Optional[bytes] = None) -> Tuple[str, str, int]:
"""
Write an uploaded file to a private temporary file.
The file is created with ``mkstemp`` (owner-only permissions, unpredictable name).
Callers are responsible for deleting it; the application only needs it on disk
while the DLP checks run.
Returns:
Tuple of (path, MIME type, size in bytes)
"""
file_name = getattr(uploaded_file, "name", None)
if file_name is None and isinstance(uploaded_file, dict):
file_name = uploaded_file.get("name", "uploaded_file")
file_name = file_name or "uploaded_file"
file_extension = os.path.splitext(file_name)[1] if "." in file_name else ""
file_bytes = content_override if content_override is not None else bytes(uploaded_file.getbuffer())
fd, file_path = tempfile.mkstemp(prefix="pcb_upload_", suffix=file_extension)
with os.fdopen(fd, "wb") as handle:
handle.write(file_bytes)
uploaded_file_type = getattr(uploaded_file, "type", None)
if uploaded_file_type is None and isinstance(uploaded_file, dict):
uploaded_file_type = uploaded_file.get("mime_type")
mime_type = uploaded_file_type or mimetypes.guess_type(file_name)[0] or "application/octet-stream"
return file_path, mime_type, len(file_bytes)
def create_new_conversation(user_id: int, title: str = "New Conversation") -> int:
"""
Create a new conversation in the database
Args:
user_id: ID of the user creating the conversation
title: Title of the conversation
Returns:
ID of the created conversation
"""
try:
with session_scope() as session:
# Create new conversation
conversation = Conversation(
user_id=user_id,
title=title
)
session.add(conversation)
# Get the new conversation ID (available after flush)
session.flush()
conversation_id = conversation.id
return conversation_id
except Exception as e:
print(f"Error creating conversation: {str(e)}")
return -1
def get_conversations(user_id: int) -> List[Dict[str, Any]]:
"""
Get all conversations for a user
Args:
user_id: ID of the user
Returns:
List of dictionaries with conversation data
"""
try:
# Get conversations with message count
from sqlalchemy import func
from sqlalchemy.orm import aliased, joinedload
# Create an alias for the Message class
Message_alias = aliased(Message)
with session_scope() as session:
# Query conversations along with their message count
conversations = session.query(Conversation)\
.options(joinedload(Conversation.messages))\
.filter(Conversation.user_id == user_id)\
.order_by(Conversation.updated_at.desc())\
.all()
# Convert to list of dictionaries to avoid detached instance issues
conversation_list = []
for conv in conversations:
message_list = []
for msg in conv.messages:
message_list.append({
"id": msg.id,
"role": msg.role,
"content": msg.content,
"timestamp": msg.timestamp
})
conversation_list.append({
"id": conv.id,
"user_id": conv.user_id,
"title": conv.title,
"created_at": conv.created_at,
"updated_at": conv.updated_at,
"messages": message_list
})
return conversation_list
except Exception as e:
print(f"Error retrieving conversations: {str(e)}")
return []
def get_conversation(conversation_id: int, requesting_user_id: Optional[int] = None, allow_admin_access: bool = False) -> Optional[Dict[str, Any]]:
"""
Get a specific conversation with all its messages
Args:
conversation_id: ID of the conversation
Returns:
Dictionary with conversation data or None if not found
"""
try:
# Use eager loading to load the messages and files
from sqlalchemy.orm import joinedload
with session_scope() as session:
query = session.query(Conversation).options(
joinedload(Conversation.messages).joinedload(Message.files)
).filter(Conversation.id == conversation_id)
if requesting_user_id is not None and not allow_admin_access:
query = query.filter(Conversation.user_id == requesting_user_id)
conversation = query.first()
if not conversation:
return None
# Convert to dictionary to avoid detached instance issues
# First, create message dictionaries including file data
message_list = []
for msg in conversation.messages:
# Convert files to dictionaries
files_list = []
for file in msg.files:
files_list.append({
"id": file.id,
"original_name": file.original_name,
"path": file.path,
"mime_type": file.mime_type,
"size": file.size,
"scan_result": file.scan_result
})
message_list.append({
"id": msg.id,
"role": msg.role,
"content": msg.content,
"timestamp": msg.timestamp,
"files": files_list
})
# Create the conversation dictionary with all data
conversation_dict = {
"id": conversation.id,
"user_id": conversation.user_id,
"title": conversation.title,
"created_at": conversation.created_at,
"updated_at": conversation.updated_at,
"messages": message_list
}
return conversation_dict
except Exception as e:
print(f"Error retrieving conversation: {str(e)}")
return None
def add_message_to_conversation(
conversation_id: int,
role: str,
content: str,
uploaded_files: Optional[List] = None
) -> Tuple[int, Optional[str]]:
"""
Add a message to a conversation
Args:
conversation_id: ID of the conversation
role: Role of the message sender ("user" or "assistant")
content: Message content
uploaded_files: List of Streamlit uploaded file objects
Returns:
Tuple containing:
- ID of the created message (0 if creation failed due to blocked files)
- Error message if any files were blocked, None otherwise
"""
saved_files: List[Dict[str, Any]] = []
def _cleanup_saved_files() -> None:
for saved in saved_files:
try:
if saved["path"] and os.path.exists(saved["path"]):
os.remove(saved["path"])
except OSError:
pass
try:
# Stage 1: persist uploads to disk and run DLP checks BEFORE touching the database.
# Returning from inside ``session_scope`` commits the transaction, so a DLP block
# discovered after the message row was added used to leave a half-saved message.
if uploaded_files:
user_id = None
if MS_DLP_AVAILABLE:
with session_scope() as session:
conversation = session.query(Conversation).filter(
Conversation.id == conversation_id
).first()
if conversation:
user_id = conversation.user_id
dlp_enabled = bool(MS_DLP_AVAILABLE and user_id and is_dlp_integration_enabled(user_id))
for uploaded_file in uploaded_files:
content_override = None
file_text = None
original_name = getattr(uploaded_file, "name", "uploaded_file")
if isinstance(uploaded_file, dict):
original_name = uploaded_file.get("name", original_name)
mime_type = uploaded_file.get("mime_type")
# DLP must inspect the original bytes: anonymized content is plain
# text and no longer carries the file's sensitivity label.
content_override = uploaded_file.get("original_bytes") or uploaded_file.get("content_bytes")
file_text = uploaded_file.get("text")
else:
mime_type = None
file_path, detected_mime_type, file_size = save_uploaded_file(uploaded_file, content_override)
mime_type = mime_type or detected_mime_type
saved_files.append({
"original_name": original_name,
"path": file_path,
"mime_type": mime_type,
"size": file_size,
})
# Check for Microsoft sensitivity labels / Purview DLP policies
if dlp_enabled:
file_allowed, dlp_error = scan_file_for_sensitivity(
user_id=user_id,
file_path=file_path,
file_name=original_name,
file_mime=mime_type,
file_text=file_text,
)
if not file_allowed:
# File blocked by DLP: nothing has been written to the database yet.
_cleanup_saved_files()
return 0, dlp_error
# Data minimisation: uploads are only needed on disk for the DLP checks.
# Their text is passed to the model in memory, so the copies are removed
# now instead of accumulating indefinitely in the temp directory.
_cleanup_saved_files()
# Stage 2: everything passed, write the message and its files atomically.
with session_scope() as session:
message = Message(
conversation_id=conversation_id,
role=role,
content=content
)
session.add(message)
session.flush()
message_id = message.id
for saved in saved_files:
session.add(File(
message_id=message_id,
original_name=saved["original_name"],
path=None, # file contents are not retained
mime_type=saved["mime_type"],
size=saved["size"],
scan_result={}
))
# Update conversation information
conversation = session.query(Conversation).filter(
Conversation.id == conversation_id
).first()
if conversation:
conversation.updated_at = func.now()
# Update conversation title based on first user message
if role == "user" and (not conversation.title or conversation.title == "New Conversation"):
# Use the first ~30 characters of the message as the title
new_title = content[:30] + "..." if len(content) > 30 else content
conversation.title = new_title
return message_id, None
except Exception as e:
_cleanup_saved_files()
print(f"Error adding message to conversation: {str(e)}")
return 0, f"Error: {str(e)}"
def delete_conversation(conversation_id: int, requesting_user_id: Optional[int] = None, allow_admin_access: bool = False) -> bool:
"""
Delete a conversation and all its messages
Args:
conversation_id: ID of the conversation to delete
Returns:
Boolean indicating success
"""
try:
with session_scope() as session:
# Find conversation
query = session.query(Conversation).filter(Conversation.id == conversation_id)
if requesting_user_id is not None and not allow_admin_access:
query = query.filter(Conversation.user_id == requesting_user_id)
conversation = query.first()
if not conversation:
return False
# Delete the conversation (cascade should handle messages and files)
session.delete(conversation)
return True
except Exception as e:
print(f"Error deleting conversation: {str(e)}")
return False
def update_user_settings(user_id: int, settings_data: Dict[str, Any]) -> bool:
"""
Update user settings
Args:
user_id: ID of the user
settings_data: Dictionary of settings to update
Returns:
Boolean indicating success
"""
# Use session_scope context manager to avoid detached instance errors
try:
with session_scope() as session:
# Find user settings
settings = session.query(Settings).filter(
Settings.user_id == user_id
).first()
if not settings:
return False
# Update settings - only allow updating attributes that exist on the model
for key, value in settings_data.items():
if hasattr(settings, key):
setattr(settings, key, value)
# session_scope handles commit and close automatically
return True
except Exception as e:
print(f"Error updating user settings: {str(e)}")
return False
def format_detection_events(events: List) -> List[Dict[str, Any]]:
"""
Format detection events for display
Args:
events: List of DetectionEvent objects
Returns:
List of formatted events with attributes extracted to avoid detached instance errors
"""
formatted_events = []
for event in events:
try:
# Extract all attributes to prevent detached instance errors
# Copy all attributes immediately to avoid further access to the detached object
event_dict = {
"id": getattr(event, 'id', 0),
"user_id": getattr(event, 'user_id', 0),
"timestamp": getattr(event, 'timestamp', None),
"action": getattr(event, 'action', "unknown"),
"severity": getattr(event, 'severity', "unknown"),
"file_names": getattr(event, 'file_names', ""),
"detected_patterns": getattr(event, 'detected_patterns', {})
}
# Format timestamp
timestamp = event_dict["timestamp"].strftime("%Y-%m-%d %H:%M:%S") if event_dict["timestamp"] else "Unknown"
event_dict["timestamp"] = timestamp
# Handle detected_patterns without calling methods on possibly detached instance
try:
# Try to access directly if possible
if isinstance(event_dict["detected_patterns"], dict):
detected_patterns = event_dict["detected_patterns"]
else:
# If it's not a dict already, try to use the JSON data directly
import json
if hasattr(event, 'detected_patterns') and event.detected_patterns is not None:
if isinstance(event.detected_patterns, str):
detected_patterns = json.loads(event.detected_patterns)
else:
detected_patterns = {}
else:
detected_patterns = {}
except Exception:
# Fallback if anything fails
detected_patterns = {}
# Calculate detection count safely
try:
detection_count = sum(len(matches) for matches in detected_patterns.values() if isinstance(matches, list))
except Exception:
detection_count = 0
# Update the dictionary with processed values
event_dict["detected_patterns"] = detected_patterns
event_dict["detection_count"] = detection_count
formatted_events.append(event_dict)
except Exception as e:
# Log the error but continue processing other events
print(f"Error formatting event: {e}")
continue
return formatted_events
def format_conversation_messages(messages: List) -> List[Dict[str, Any]]:
"""
Format conversation messages for AI processing
Args:
messages: List of Message objects or message dictionaries
Returns:
List of formatted messages with attributes extracted to avoid detached instance errors,
sorted by timestamp
"""
formatted_messages = []
for message in messages:
try:
# Check if the message is already a dictionary
if isinstance(message, dict):
# Message is already a dictionary, use it directly
message_dict = {
"id": message.get('id', 0),
"conversation_id": message.get('conversation_id', 0),
"role": message.get('role', "user"),
"content": message.get('content', ""),
"timestamp": message.get('timestamp', None)
}
else:
# Extract all attributes to prevent detached instance errors
message_dict = {
"id": getattr(message, 'id', 0),
"conversation_id": getattr(message, 'conversation_id', 0),
"role": getattr(message, 'role', "user"),
"content": getattr(message, 'content', ""),
"timestamp": getattr(message, 'timestamp', None)
}
formatted_messages.append(message_dict)
except Exception as e:
# Log the error but continue processing other messages
print(f"Error formatting message: {e}")
continue
# Sort messages by timestamp
formatted_messages.sort(key=lambda x: x["timestamp"] if x["timestamp"] else datetime.min)
return formatted_messages