Fix provider configuration and model handling issues

.env.example

@@ -1,33 +1,42 @@
+# ================================================================================
 # .env.example — Environment variables for LLM Strata
 # Copy this to .env and fill in the values for your active provider.
 # You only need the keys for the provider set in profiles.yaml.
+# ================================================================================
 
-# ── OpenAI (provider: openai) ────────────────────────────────────────
+# — OpenAI (provider: openai) ─────────
 OPENAI_API_KEY=sk-...
+# OPENAI_CHAT_ENDPOINT=https://api.openai.com/v1
 
-# ── Azure OpenAI (provider: azure) ──────────────────────────────────
-AZURE_OPENAI_API_KEY=...
-AZURE_OPENAI_ENDPOINT=https://your-resource.openai.azure.com/
-AZURE_OPENAI_API_VERSION=2024-12-01-preview
+# — Azure OpenAI (provider: azure) ─────────────────────
+# AZURE_OPENAI_API_KEY=...
+# AZURE_OPENAI_ENDPOINT=https://your-resource.openai.azure.com/
+# AZURE_OPENAI_API_VERSION=2024-12-01-preview
 
-# ── Google Gemini (provider: gemini) ────────────────────────────────
-GEMINI_API_KEY=AIza...                 # free at aistudio.google.com
+# — Google Gemini (provider: gemini) ─────────────────────────────
+# GEMINI_API_KEY=AIza...                           # free at aistudio.google.com
+# GEMINI_ENDPOINT=https://generativelanguage.googleapis.com/v1beta/openai/
 
-# ── Anthropic (provider: anthropic) — also requires: pip install litellm ───
-ANTHROPIC_API_KEY=sk-ant-...
+# — Anthropic (provider: anthropic) — also requires: pip install litellm ──────
+# ANTHROPIC_API_KEY=sk-ant-...
 
-# ── AWS Bedrock (provider: bedrock) — also requires: pip install litellm boto3
-AWS_ACCESS_KEY_ID=AKIA...
-AWS_SECRET_ACCESS_KEY=...
-AWS_REGION_NAME=us-east-1
+# — AWS Bedrock (provider: bedrock) — also requires: pip install litellm boto3
+# AWS_ACCESS_KEY_ID=AKIA...
+# AWS_SECRET_ACCESS_KEY=...
+# AWS_REGION_NAME=us-east-1
 
-# ── Hugging Face (provider: huggingface) ─────────────────────────────
-HF_TOKEN=hf_...                         # huggingface.co/settings/tokens
+# — Hugging Face (provider: huggingface) ─────────────────────
+# HF_TOKEN=hf_...                                  # huggingface.co/settings/tokens
+# HF_ENDPOINT=https://api-inference.huggingface.co/v1/
 
-# ── Ollama (provider: ollama) — no key needed, local only ───────────
-OLLAMA_BASE_URL=http://localhost:11434/v1  # optional, this is the default
+# — Ollama (provider: ollama) — no key needed, local only ─────────────
+# OLLAMA_BASE_URL=http://localhost:11434/v1
 
-# ── LangFuse (all providers — for production monitoring) ────────────
-LANGFUSE_PUBLIC_KEY=pk-...
+# — Private/on-prem OpenAI-compatible server ─────────────────
+# OPENAI_COMPATIBLE_BASE_URL=http://localhost:8000/v1
+# OPENAI_COMPATIBLE_API_KEY=local-key              # only when api_key_required: true
+
+# — LangFuse (all providers — for production monitoring) ─────────────────
+LANGFUSE_PUBLIC_KEY=pk-lf-...
 LANGFUSE_SECRET_KEY=sk-lf-...
-LANGFUSE_BASE_URL=https://cloud.langfuse.com
+LANGFUSE_BASE_URL=https://cloud.langfuse.com

README.md

@@ -60,7 +60,7 @@ LLM Strata is a collection of independent security and safety test layers. The l
 
 LLM Strata is composed of 14 independent security and safety layers. Each layer has its own runner, configuration, outputs, and direct command, so it can be executed independently or as part of the full pipeline. The layers share provider, profile, and reporting utilities while remaining independently configurable.
 
-The framework combines pre-deployment testing, runtime scanning, agent/RAG security tests, human review, and production monitoring. Coverage is configurable and should be interpreted as evidence from the selected test corpus—not as a guarantee of complete protection.
+The framework combines pre-deployment testing, runtime scanning, agent/RAG security tests, human review, and production monitoring. Coverage is configurable and should be interpreted as evidence from the selected test corpus.
 
 - 🧪 **Pre-deployment:** [Garak](garak/garak.md), [DeepEval](deepeval/deepeval.md), [RAGAS](ragas/ragas.md), [PyRIT](pyrit/pyrit.md)
 - 📡 **Runtime and monitoring:** [LLM Guard](llm_guard/llm-guard.md), [LangFuse](langfuse/langfuse.md)

client_factory.py

@@ -72,15 +72,23 @@ def get_client(provider: str | None = None):
 
     elif provider == "gemini":
         from openai import OpenAI
+        base_url = os.environ.get(
+            "GEMINI_ENDPOINT",
+            "https://generativelanguage.googleapis.com/v1beta/openai/",
+        )
         return OpenAI(
-            base_url="https://generativelanguage.googleapis.com/v1beta/openai/",
+            base_url=base_url,
             api_key=os.environ["GEMINI_API_KEY"],
         )
 
     elif provider == "huggingface":
         from openai import OpenAI
+        base_url = os.environ.get(
+            "HF_ENDPOINT",
+            "https://api-inference.huggingface.co/v1/",
+        )
         return OpenAI(
-            base_url="https://api-inference.huggingface.co/v1/",
+            base_url=base_url,
             api_key=os.environ["HF_TOKEN"],
         )
 

deepeval/run_deepeval.py

@@ -73,6 +73,7 @@ def create_deepeval_model(model_name: str):
     class ProviderModel(DeepEvalBaseLLM):
         def __init__(self):
             self.model_name = model_name
+            self.name = model_name   # required by DeepEvalBaseLLM; super().__init__() is skipped
             self.client = get_client()
             self.model = self.load_model()
 
@@ -100,7 +101,10 @@ def generate(self, prompt: str, schema=None):
                 start, end = payload.find("{"), payload.rfind("}")
                 if start < 0 or end <= start:
                     raise ValueError("Evaluator returned invalid structured output.") from None
-                data = json.loads(payload[start:end + 1])
+                try:
+                    data = json.loads(payload[start:end + 1])
+                except json.JSONDecodeError:
+                    raise ValueError("Evaluator returned invalid structured output.") from None    
 
             if hasattr(schema, "model_validate"):
                 return schema.model_validate(data)
@@ -115,8 +119,24 @@ def get_model_name(self):
     return ProviderModel()
 
 
+def _resolve_evaluator(evaluator_model):
+    """Return evaluator_model unchanged if already set; otherwise build one from the active profile.
+
+    Ensures that runner functions called directly (not via run_all_tests) always
+    use the profile-resolved evaluator model rather than falling through to
+    DeepEval's internal default.
+    """
+    if evaluator_model is not None:
+        return evaluator_model
+    config = load_config()
+    return create_deepeval_model(
+        get_model("evaluator_model", config.get("evaluator_model", "gpt-4o-mini"))
+    )
+
+
 def run_bias_test(model_name: str, test_input: str, verbose: bool,
-                  evaluator_model: str = "gpt-4o-mini", threshold: float = 0.5) -> dict:
+                  evaluator_model = None, threshold: float = 0.5) -> dict:
+    evaluator_model = _resolve_evaluator(evaluator_model)            
     from deepeval.metrics import BiasMetric
     from deepeval.test_case import LLMTestCase
 
@@ -145,7 +165,8 @@ def run_bias_test(model_name: str, test_input: str, verbose: bool,
 
 
 def run_toxicity_test(model_name: str, test_input: str, verbose: bool,
-                      evaluator_model: str = "gpt-4o-mini", threshold: float = 0.5) -> dict:
+                      evaluator_model = None, threshold: float = 0.5) -> dict:
+    evaluator_model = _resolve_evaluator(evaluator_model)            
     from deepeval.metrics import ToxicityMetric
     from deepeval.test_case import LLMTestCase
 
@@ -174,7 +195,8 @@ def run_toxicity_test(model_name: str, test_input: str, verbose: bool,
 
 
 def run_hallucination_test(model_name: str, test_input: str, verbose: bool,
-                           evaluator_model: str = "gpt-4o-mini", threshold: float = 0.5) -> dict:
+                           evaluator_model = None, threshold: float = 0.5) -> dict:
+    evaluator_model = _resolve_evaluator(evaluator_model)            
     from deepeval.metrics import HallucinationMetric
     from deepeval.test_case import LLMTestCase
 
@@ -207,7 +229,8 @@ def run_hallucination_test(model_name: str, test_input: str, verbose: bool,
 
 
 def run_misuse_test(model_name: str, test_input: str, verbose: bool,
-                    evaluator_model: str = "gpt-4o-mini", threshold: float = 0.5) -> dict:
+                    evaluator_model = None, threshold: float = 0.5) -> dict:
+    evaluator_model = _resolve_evaluator(evaluator_model)            
     from deepeval.metrics import MisuseMetric
     from deepeval.test_case import LLMTestCase
 
@@ -236,7 +259,8 @@ def run_misuse_test(model_name: str, test_input: str, verbose: bool,
 
 
 def run_pii_test(model_name: str, test_input: str, verbose: bool,
-                 evaluator_model: str = "gpt-4o-mini", threshold: float = 0.5) -> dict:
+                 evaluator_model = None, threshold: float = 0.5) -> dict:
+    evaluator_model = _resolve_evaluator(evaluator_model)            
     from deepeval.metrics import PIILeakageMetric
     from deepeval.test_case import LLMTestCase
 

in_context_attacks/run_many_shot.py

@@ -90,17 +90,15 @@ def build_shot_list(
         harmful = _cycle(harmful_shots, n_harmful)
         return benign + harmful
 
-    if strategy == "camouflaged":
-        n_harmful = max(1, int(shot_count * 0.2))
-        n_benign  = shot_count - n_harmful
-
-        benign  = _cycle(benign_shots, n_benign)
-        harmful = _cycle(harmful_shots, n_harmful)
-        combined = benign + harmful
-        random.shuffle(combined)
-        return combined
-
-    raise ValueError(f"Unknown strategy: {strategy!r}. Choose from {VALID_STRATEGIES}")
+    # strategy == "camouflaged":
+    n_harmful = max(1, int(shot_count * 0.2))
+    n_benign  = shot_count - n_harmful
+
+    benign  = _cycle(benign_shots, n_benign)
+    harmful = _cycle(harmful_shots, n_harmful)
+    combined = benign + harmful
+    random.shuffle(combined)
+    return combined
 
 
 # — Prompt builder ————————————————————————————————

langfuse/run_langfuse.py

@@ -80,13 +80,13 @@ def get_langfuse_client(config: dict):
     Reads LANGFUSE_PUBLIC_KEY, LANGFUSE_SECRET_KEY, LANGFUSE_BASE_URL from env.
     Falls back to config host if LANGFUSE_BASE_URL is not set.
     """
-    from langfuse import get_client
+    from langfuse import get_client as _langfuse_get_client
 
     host = config.get("host", "https://cloud.langfuse.com")
     if "LANGFUSE_BASE_URL" not in os.environ:
         os.environ["LANGFUSE_BASE_URL"] = host
 
-    return get_client()
+    return _langfuse_get_client()
 
 
 def verify_connection(config: dict) -> bool:

llm_guard/guard_config.yaml

@@ -2,18 +2,19 @@
 # LLM Guard - Runtime Input/Output Protection Configuration
 # Docs: https://llm-guard.com
 # ==============================================================================
-
+#
 # WHAT IS LLM GUARD?
 #   LLM Guard sits between your user and the LLM at runtime.
 #   It scans every request (input) and every response (output) in real-time,
 #   blocking or sanitizing anything dangerous before it causes harm.
-
+#
 # HOW IT WORKS:
 #   User Request -> [Input Scanners] -> LLM -> [Output Scanners] -> Safe Response
-
+#
 # HOW TO RUN:
 #   python llm_guard/run_guard.py          # demo mode (test scanners)
 #   python llm_guard/app.py                # start as a proxy API server
+#=================================================================================
 
 # ----------------------- Input Scanners--------------------------------------------
 # Applied to every user message BEFORE it reaches the LLM.
@@ -99,6 +100,10 @@ output_scanners:
     enabled: true
     threshold: 0.5           # response must be at least 50% relevant to input
 
+
+# ---- Model (overridden by active profile in profiles.yaml)
+model: gpt-4o-mini                     # model to call for allowed prompts
+
 # ---------------------------- Global Settings ---------------------------- 
 settings:
   # What to do when a scanner triggers:
@@ -109,4 +114,4 @@ settings:
 
   # Log all scanner results (even passes) for audit trail
   audit_logging: true
-  log_file: results/llm_guard_audit.log
+  log_file: results/llm_guard_audit.log

llm_guard/run_guard.py

@@ -31,21 +31,30 @@
 from env_loader import load_dotenv
 load_dotenv()
 
+from profile_loader import get_model
+
 try:
     from client_factory import get_client
     client = get_client()
     OPENAI_AVAILABLE = True
 except Exception:
+    client = None
     OPENAI_AVAILABLE = False
 
 
 PROJECT_ROOT = Path(__file__).resolve().parent.parent
 CONFIG_PATH = PROJECT_ROOT / "llm_guard/guard_config.yaml"
 
 
+_CONFIG_CACHE: dict | None = None
+
+
 def load_config() -> dict:
-    with open(CONFIG_PATH) as f:
-        return yaml.safe_load(f)
+    global _CONFIG_CACHE
+    if _CONFIG_CACHE is None:
+        with open(CONFIG_PATH) as f:
+            _CONFIG_CACHE = yaml.safe_load(f)
+    return _CONFIG_CACHE
 
 
 # ── PII patterns (regex + replacement placeholder) ──────────────────────────
@@ -346,8 +355,11 @@ def scan_input(prompt: str) -> tuple:
     return sanitized, is_valid, results_score
 
 
-def get_llm_response(prompt: str, model: str = "gpt-4o-mini") -> str:
-    """Call OpenAI API. Returns mock response if API key not set."""
+def get_llm_response(prompt: str, model: str | None = None) -> str:
+    """Call the active profile's LLM. Returns mock response if client unavialable"""
+    if model is None:
+        config = load_config()
+        model =get_model("model",config.get("model","gpt-4o-mini"))
     if not OPENAI_AVAILABLE:
         return f"[MOCK RESPONSE] This is a simulated response to: '{prompt[:50]}...'"
     response = client.chat.completions.create(

multilingual/run_multilingual.py

@@ -46,8 +46,7 @@
 
 # --- Prompt builders -----------------------------------------------
 
-def build_prompt(technique: str, lang_code: str, lang_name: str,
-                  translated_text: str, english_text: str) -> str:
+def build_prompt(technique: str, lang_name: str, translated_text: str) -> str:
     if technique == "direct":
         return translated_text
 
@@ -131,7 +130,6 @@ def run_multilingual_tests(
         category = attack["category"]
         intent   = attack["intent"]
         translations = attack.get("translations", {})
-        english_text = translations.get("en", "")
 
         for lang in active_languages:
             lang_code = lang["code"]
@@ -142,8 +140,7 @@ def run_multilingual_tests(
 
             for technique in active_techniques:
                 try:
-                    prompt = build_prompt(technique, lang_code, lang_name,
-                        translated, english_text)
+                    prompt = build_prompt(technique, lang_name, translated)
                     resp = client.chat.completions.create(
                         model=model_name,
                         messages=[{"role": "user", "content": prompt}],

profile_loader.py

@@ -55,6 +55,7 @@ def load_profile() -> dict:
 
 def get_model(key: str, fallback: str) -> str:
     """Resolve a model name from the active profile, falling back to the module config value.
+
     Args:
         key:      Profile key to look up (e.g. 'model', 'judge_model', 'attacker_model').
         fallback: Value from the module's YAML config (or hardcoded default) to use
@@ -73,7 +74,6 @@ def active_profile_name() -> str:
         return "none"
     if _active_name_cache is not None:
         return _active_name_cache
-    with open(_PROFILES_PATH) as f:
-        data = yaml.safe_load(f) or {}
-    return os.environ.get("ACTIVE_PROFILE") or data.get("active_profile", "cost_optimized")
+    load_profile()  # populates _active_name_cache as a sideeffect from global declartion
+    return _active_name_cache or "cost_optimized"