Repository navigation
Expand file tree
/
Copy pathdatabase_check.py
More file actions
145 lines (121 loc) · 4.74 KB
/
Copy pathdatabase_check.py
File metadata and controls
145 lines (121 loc) · 4.74 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
#!/usr/bin/env python3
"""
Database Schema Validation Script
This script checks the database schema to ensure all required columns are present
and runs any missing migrations if needed.
"""
import os
import sys
import logging
from sqlalchemy import inspect
from database import init_db, get_session
from models import Settings
# Configure logging
logging.basicConfig(
level=logging.INFO,
format='%(asctime)s - %(name)s - %(levelname)s - %(message)s'
)
logger = logging.getLogger("database_check")
# List of required columns for the Settings table
DLP_COLUMNS = [
"enable_ms_dlp",
"ms_dlp_sensitivity_threshold"
]
LOCAL_LLM_COLUMNS = [
"local_model_path",
"local_model_context_size",
"local_model_gpu_layers",
"local_model_temperature"
]
def check_columns_exist(table_name, column_list):
"""
Check if all columns in the list exist in the specified table
Args:
table_name: Name of the table to check
column_list: List of column names to check for
Returns:
Tuple of (all_exist, missing_columns)
"""
session = get_session()
if session is None:
raise RuntimeError("Unable to create a database session; is DATABASE_URL configured?")
inspector = inspect(session.bind)
try:
existing_columns = [col['name'] for col in inspector.get_columns(table_name)]
missing_columns = [col for col in column_list if col not in existing_columns]
all_exist = len(missing_columns) == 0
return all_exist, missing_columns
finally:
session.close()
def run_migration(migration_module_name):
"""
Import and run a migration module
Args:
migration_module_name: Name of the migration module
Returns:
True if migration succeeded, False otherwise
"""
try:
migration_module = __import__(migration_module_name)
migration_module.run_migration()
logger.info(f"Successfully ran migration: {migration_module_name}")
return True
except Exception as e:
logger.error(f"Failed to run migration {migration_module_name}: {e}")
return False
def main():
"""Main function to check database schema and run migrations if needed"""
logger.info("Initializing database connection...")
init_db()
# Check DLP columns
logger.info("Checking Microsoft DLP columns...")
dlp_columns_exist, missing_dlp_columns = check_columns_exist("settings", DLP_COLUMNS)
if not dlp_columns_exist:
logger.warning(f"Missing DLP columns: {missing_dlp_columns}")
logger.info("Running DLP migration...")
if run_migration("migration_add_dlp_columns"):
logger.info("DLP migration successful")
else:
logger.error("DLP migration failed")
else:
logger.info("DLP columns exist")
# Check Local LLM columns
logger.info("Checking Local LLM columns...")
llm_columns_exist, missing_llm_columns = check_columns_exist("settings", LOCAL_LLM_COLUMNS)
if not llm_columns_exist:
logger.warning(f"Missing Local LLM columns: {missing_llm_columns}")
logger.info("Running Local LLM migration...")
if run_migration("migration_add_local_llm_columns"):
logger.info("Local LLM migration successful")
else:
logger.error("Local LLM migration failed")
else:
logger.info("Local LLM columns exist")
# Run pattern levels migration to ensure all custom patterns have level attribute
logger.info("Running pattern levels migration...")
if run_migration("migration_pattern_levels"):
logger.info("Pattern levels migration successful")
else:
logger.error("Pattern levels migration failed")
# Encrypt legacy plaintext data (when a key is configured) and mask old audit values
logger.info("Securing existing data...")
if run_migration("migration_secure_existing_data"):
logger.info("Existing data secured")
else:
logger.error("Securing existing data failed")
# Final check after migrations
dlp_columns_exist, missing_dlp_columns = check_columns_exist("settings", DLP_COLUMNS)
llm_columns_exist, missing_llm_columns = check_columns_exist("settings", LOCAL_LLM_COLUMNS)
all_ok = dlp_columns_exist and llm_columns_exist
if all_ok:
logger.info("All database columns are properly configured")
return 0
else:
logger.error("Some columns are still missing in the database schema")
if not dlp_columns_exist:
logger.error(f"Missing DLP columns: {missing_dlp_columns}")
if not llm_columns_exist:
logger.error(f"Missing Local LLM columns: {missing_llm_columns}")
return 1
if __name__ == "__main__":
sys.exit(main())