diff --git a/aiperf/Dockerfile b/aiperf/Dockerfile
index ffe1551..05d5897 100644
--- a/aiperf/Dockerfile
+++ b/aiperf/Dockerfile
@@ -2,6 +2,12 @@ FROM python:3.11-slim
 
 # Install AIPerf for performance benchmarking
 # Install Datadog for metrics export (Custom Metrics API)
+#
+# NOTE: these are intentionally unpinned upstream, so an image rebuilt tomorrow
+# may not contain the same code as the one running today, and a compromised
+# release would be pulled in automatically. Pinning requires resolving the exact
+# versions against PyPI (`pip install aiperf==X.Y.Z datadog==A.B.C`), which needs
+# network access this change does not have; recorded here as the next step.
 RUN pip install --no-cache-dir aiperf datadog
 
 WORKDIR /scripts
@@ -13,4 +19,14 @@ COPY common/ /scripts/common/
 COPY aiperf/benchmark.py /scripts/benchmark.py
 RUN chmod +x /scripts/benchmark.py
 
+# The image ran as root: the benchmark needs no privileges, but any code-execution
+# bug in it (or in a dependency) started as uid 0 inside the container, which is
+# the precondition for most container escapes. A dedicated unprivileged account is
+# created instead. The GID is matched by `fsGroup` in cronjob.yaml so the mounted
+# results volume stays writable.
+RUN groupadd --gid 10001 benchmark \
+    && useradd --uid 10001 --gid 10001 --create-home --shell /usr/sbin/nologin benchmark \
+    && chown -R benchmark:benchmark /scripts
+USER 10001:10001
+
 CMD ["python3", "/scripts/benchmark.py"]
diff --git a/aiperf/benchmark.py b/aiperf/benchmark.py
index da1f95c..2f1258c 100755
--- a/aiperf/benchmark.py
+++ b/aiperf/benchmark.py
@@ -1,535 +1,688 @@
-#!/usr/bin/env python3
-"""
-AIPerf Benchmark Script for Inference Endpoints
-Runs performance benchmarks against inference endpoints using AIPerf
-"""
-
-import os
-import sys
-import subprocess
-import json
-import time
-import threading
-from pathlib import Path
-from datetime import datetime
-from typing import Optional, Dict, Any, List
-import re
-
-# Add common directory to path
-sys.path.insert(0, '/scripts/common')
-from datadog_utils import send_metrics_async
-
-
-def _normalize_endpoint_url(url: str) -> str:
-    """Normalize endpoint URL to base only so AIPerf does not double-append path.
-
-    AIPerf appends the endpoint path (e.g. /v1/chat/completions) to --url. If --url
-    already contains that path, the request goes to /v1/chat/completions/v1/chat/completions → 404.
-    Strip known paths so both "https://api.hyperbolic.xyz" and
-    "https://api.hyperbolic.xyz/v1/chat/completions" result in base URL only.
-    """
-    url = (url or "").rstrip("/")
-    # Strip trailing path if it matches OpenAI-style chat/completions path
-    for suffix in ("/v1/chat/completions", "/v1/completions", "/v1/embeddings"):
-        if url.lower().endswith(suffix):
-            return url[: -len(suffix)].rstrip("/") or url
-    return url
-
-
-def run_benchmark(
-    model_name: str,
-    endpoint_url: str,
-    endpoint_type: str = "chat",
-    concurrency: int = 20,
-    request_count: int = 100,
-    streaming: bool = True,
-    output_dir: str = "/tmp/aiperf-results",
-    benchmark_duration: Optional[int] = None,
-    benchmark_grace_period: Optional[int] = None,
-    request_timeout_seconds: Optional[float] = None,
-    output_tokens_mean: Optional[int] = None,
-    cf_access_client_id: Optional[str] = None,
-    cf_access_client_secret: Optional[str] = None,
-    api_key: Optional[str] = None,
-    **kwargs
-) -> Dict[str, Any]:
-    """
-    Run AIPerf benchmark against an inference endpoint.
-    
-    Args:
-        model_name: Model identifier for AIPerf (must be valid HuggingFace id, e.g. "Qwen/Qwen3-VL-32B-Thinking")
-        endpoint_url: Chat endpoint URL (e.g. https://api.hyperbolic.xyz/v1/chat/completions; model is passed in request body)
-        endpoint_type: Type of endpoint (chat, completions, embeddings)
-        concurrency: Number of concurrent requests
-        request_count: Total number of requests
-        streaming: Enable streaming
-        output_dir: Directory for results
-        output_tokens_mean: Mean number of output tokens per response
-        cf_access_client_id: Cloudflare Access Client ID (optional, for inference.hyperbolic.xyz)
-        cf_access_client_secret: Cloudflare Access Client Secret (optional)
-        api_key: API key for Bearer auth (optional, for api.hyperbolic.xyz)
-        **kwargs: Additional AIPerf arguments
-    
-    Returns:
-        Dictionary with benchmark results and metadata
-    """
-    print("=" * 60)
-    print("AIPerf Benchmark")
-    print("=" * 60)
-    print(f"Model: {model_name}")
-    print(f"Endpoint: {endpoint_url}")
-    print(f"Type: {endpoint_type}")
-    print(f"Concurrency: {concurrency}")
-    if benchmark_duration:
-        print(f"Duration: {benchmark_duration}s")
-        if benchmark_grace_period:
-            print(f"Grace Period: {benchmark_grace_period}s")
-    else:
-        print(f"Request Count: {request_count}")
-    if output_tokens_mean:
-        print(f"Output Tokens Mean: {output_tokens_mean}")
-    print(f"Streaming: {streaming}")
-    print(f"Output Directory: {output_dir}")
-    if api_key:
-        print(f"Auth: API key (Bearer)")
-    elif cf_access_client_id:
-        print(f"Cloudflare Access: Enabled (Client ID: {cf_access_client_id[:20]}...)")
-    print("=" * 60)
-    print()
-    
-    # Create output directory
-    output_path = Path(output_dir)
-    output_path.mkdir(parents=True, exist_ok=True)
-    
-    # Build AIPerf command
-    cmd = [
-        "aiperf", "profile",
-        "--model", model_name,
-        "--url", endpoint_url,
-        "--endpoint-type", endpoint_type,
-        "--concurrency", str(concurrency),
-        "--output-artifact-dir", output_dir,
-        "--ui-type", "none",  # Disable TUI for non-interactive environments (Kubernetes/containers)
-        "--no-server-metrics",  # Disable server metrics to avoid timeout on unreachable Prometheus endpoints (known issue in v0.4.0+)
-    ]
-    
-    # AIPerf doesn't allow both --request-count and --benchmark-duration together
-    if benchmark_duration is not None:
-        cmd.extend(["--benchmark-duration", str(benchmark_duration)])
-        if benchmark_grace_period is not None:
-            cmd.extend(["--benchmark-grace-period", str(benchmark_grace_period)])
-    else:
-        cmd.extend(["--request-count", str(request_count)])
-    
-    if streaming:
-        cmd.append("--streaming")
-    
-    if request_timeout_seconds is not None:
-        cmd.extend(["--request-timeout-seconds", str(request_timeout_seconds)])
-    
-    if output_tokens_mean is not None:
-        cmd.extend(["--output-tokens-mean", str(output_tokens_mean)])
-        # Send max_tokens in request body (not max_completion_tokens) so backends and gateways apply the limit
-        cmd.append("--use-legacy-max-tokens")
-    
-    # Add auth: API key (Bearer) for api.hyperbolic.xyz, else Cloudflare Access for inference.hyperbolic.xyz
-    if api_key:
-        cmd.extend(["--header", f"Authorization: Bearer {api_key}"])
-    elif cf_access_client_id and cf_access_client_secret:
-        cmd.extend(["--header", f"CF-Access-Client-Id: {cf_access_client_id}"])
-        cmd.extend(["--header", f"CF-Access-Client-Secret: {cf_access_client_secret}"])
-    
-    # Add any additional kwargs
-    for key, value in kwargs.items():
-        if value is not None:
-            flag_name = key.replace('_', '-')
-            cmd.extend([f"--{flag_name}", str(value)])
-    
-    # Redact API key in logged command (avoid leaking secrets)
-    cmd_safe = []
-    for part in cmd:
-        if "Authorization:" in part and "Bearer " in part and "***" not in part:
-            cmd_safe.append(part.split("Bearer ", 1)[0] + "Bearer ***")
-        else:
-            cmd_safe.append(part)
-    print(f"Running command: {' '.join(cmd_safe)}")
-    print()
-    
-    # Prepare environment variables to disable TUI in non-interactive environments
-    env = os.environ.copy()
-    env['TERM'] = 'dumb'  # Disable terminal capabilities
-    env['CI'] = 'true'    # Signal CI/non-interactive environment
-    env['NO_COLOR'] = '1'  # Disable ANSI color codes
-    env['PYTHONUNBUFFERED'] = '1'  # Disable Python output buffering
-    
-    # Set AIPerf timeout environment variables to prevent timeout errors
-    # These timeouts control how long AIPerf waits for services to respond during startup
-    # Increase them for slower environments (e.g., Kubernetes with resource constraints)
-    env.setdefault('AIPERF_SERVICE_PROFILE_CONFIGURE_TIMEOUT', '600.0')  # 10 minutes for configuration
-    env.setdefault('AIPERF_SERVICE_PROFILE_START_TIMEOUT', '300.0')  # 5 minutes for start profiling
-    env.setdefault('AIPERF_DATASET_CONFIGURATION_TIMEOUT', '600.0')  # 10 minutes for dataset config
-    
-    try:
-        # Run AIPerf with --ui-type none to disable TUI completely
-        # This is the proper way to run AIPerf in non-interactive environments (Kubernetes/containers)
-        # With --ui-type none, AIPerf should exit cleanly without TUI-related errors
-        result = subprocess.run(
-            cmd,
-            stdin=subprocess.DEVNULL,  # Redirect stdin to prevent any TUI detection
-            stdout=subprocess.PIPE,    # Use PIPE (not TTY) - makes isatty() return False
-            stderr=subprocess.STDOUT,  # Merge stderr into stdout to capture all output
-            text=True,
-            check=True,  # With --ui-type none, AIPerf should exit cleanly
-            env=env,  # Pass environment with TUI-disabling variables
-            timeout=None  # No timeout - let benchmark run for full duration
-        )
-        
-        print("✅ Benchmark completed successfully")
-        print(f"Results saved to: {output_dir}")
-        
-        # Parse results if available; redact API key from stdout (AIPerf may echo full CLI command)
-        stdout_safe = result.stdout
-        if api_key and api_key in stdout_safe:
-            stdout_safe = stdout_safe.replace(api_key, "***REDACTED***")
-        # Also redact Bearer sk_... patterns in case key appears in other forms
-        stdout_safe = re.sub(r"Bearer\s+sk_[^\s'\"]+", "Bearer ***", stdout_safe, flags=re.IGNORECASE)
-
-        results = {
-            "status": "success",
-            "timestamp": datetime.utcnow().isoformat(),
-            "model": model_name,
-            "endpoint": endpoint_url,
-            "output_dir": output_dir,
-            "stdout": stdout_safe,
-        }
-        
-        # Try to find and parse result files
-        result_files = list(output_path.glob("*.json*"))
-        if result_files:
-            results["result_files"] = [str(f) for f in result_files]
-        
-        return results
-        
-    except subprocess.CalledProcessError as e:
-        print(f"❌ Benchmark failed with exit code {e.returncode}")
-        
-        # Check if result files exist despite the error (partial success)
-        result_files = list(output_path.glob("*.json*"))
-        if result_files:
-            print(f"⚠️  Warning: Exit code {e.returncode}, but result files found:")
-            for f in result_files:
-                print(f"   - {f}")
-            print("   This may indicate a non-fatal error (e.g., timeout during cleanup)")
-        
-        # Show error output (truncated for readability)
-        stdout_preview = e.stdout[:1000] if e.stdout else '(empty)'
-        stderr_preview = e.stderr[:1000] if e.stderr else '(empty)'
-        print(f"\nStdout preview (first 1000 chars):\n{stdout_preview}")
-        if e.stderr and e.stderr != e.stdout:
-            print(f"\nStderr preview (first 1000 chars):\n{stderr_preview}")
-        
-        # Check for common error patterns
-        error_msg = e.stderr or e.stdout or "Unknown error"
-        if "TimeoutError" in error_msg or "timeout" in error_msg.lower():
-            print("\n💡 Tip: If you see timeout errors, try increasing:")
-            print("   - AIPERF_SERVICE_PROFILE_START_TIMEOUT")
-            print("   - AIPERF_SERVICE_PROFILE_CONFIGURE_TIMEOUT")
-            print("   - AIPERF_DATASET_CONFIGURATION_TIMEOUT")
-        
-        stdout_safe = (e.stdout or "")
-        if api_key and api_key in stdout_safe:
-            stdout_safe = stdout_safe.replace(api_key, "***REDACTED***")
-        stdout_safe = re.sub(r"Bearer\s+sk_[^\s'\"]+", "Bearer ***", stdout_safe, flags=re.IGNORECASE)
-        return {
-            "status": "error",
-            "timestamp": datetime.utcnow().isoformat(),
-            "error": error_msg,
-            "exit_code": e.returncode,
-            "stdout": stdout_safe,
-            "stderr": e.stderr,
-            "result_files": [str(f) for f in result_files] if result_files else None,
-        }
-    except FileNotFoundError:
-        print("❌ AIPerf not found. Please install it with: pip install aiperf")
-        return {
-            "status": "error",
-            "timestamp": datetime.utcnow().isoformat(),
-            "error": "AIPerf not installed",
-        }
-
-
-def parse_aiperf_results(result_dir: str) -> Dict[str, float]:
-    """
-    Parse AIPerf JSON result files.
-    
-    AIPerf exports metrics in profile_export_aiperf.json with structure:
-    {
-        "schema_version": "1.0",
-        "aiperf_version": "...",
-        "benchmark_id": "...",
-        "request_latency": {
-            "unit": "ms",
-            "avg": 762.60,
-            "p50": 752.52,
-            "p95": 790.74,
-            "p99": 1013.11,
-            "min": 705.02,
-            "max": 1725.33,
-            "std": 44.68,
-            ...
-        },
-        "time_to_first_token": { ... },
-        ...
-    }
-    
-    Each metric field is a JsonMetricResult object with: unit, avg, p1, p5, p10, p25, p50, p75, p90, p95, p99, min, max, std
-    
-    Args:
-        result_dir: Directory containing JSON result files
-    
-    Returns:
-        Dictionary of metric_name_stat -> value (e.g., "request_latency_avg", "time_to_first_token_p99")
-    """
-    result_path = Path(result_dir)
-    metrics = {}
-    
-    # Look specifically for profile_export_aiperf.json (the aggregated stats file)
-    json_file = result_path / "profile_export_aiperf.json"
-    
-    if not json_file.exists():
-        print(f"⚠️  No profile_export_aiperf.json found in {result_dir}")
-        return metrics
-    
-    try:
-        with open(json_file) as f:
-            data = json.load(f)
-        
-        if not isinstance(data, dict):
-            print(f"⚠️  Expected dict in {json_file}, got {type(data)}")
-            return metrics
-        
-        # AIPerf JsonExportData structure: metric fields are top-level keys containing JsonMetricResult objects
-        # JsonMetricResult has: unit, avg, p1, p5, p10, p25, p50, p75, p90, p95, p99, min, max, std
-        
-        # Known non-metric fields to skip
-        non_metric_fields = {
-            "schema_version", "aiperf_version", "benchmark_id", "input_config",
-            "was_cancelled", "error_summary", "start_time", "end_time", "telemetry_data"
-        }
-        
-        # All possible stat fields in JsonMetricResult
-        stat_fields = ["avg", "p1", "p5", "p10", "p25", "p50", "p75", "p90", "p95", "p99", "min", "max", "std"]
-        
-        # Extract metrics from all top-level keys that are JsonMetricResult objects
-        for key, value in data.items():
-            # Skip known non-metric fields
-            if key in non_metric_fields:
-                continue
-            
-            # Check if this is a JsonMetricResult (has "unit" field)
-            if isinstance(value, dict) and "unit" in value:
-                # Extract all available stats from this JsonMetricResult
-                for stat in stat_fields:
-                    if stat in value and value[stat] is not None:
-                        try:
-                            metrics[f"{key}_{stat}"] = float(value[stat])
-                        except (ValueError, TypeError):
-                            pass
-        
-        # Extract error_summary so we send request/error counts even when 0 successful requests
-        # (dashboard can then show "last run had X errors" instead of empty)
-        error_summary = data.get("error_summary")
-        if isinstance(error_summary, list):
-            total_errors = 0
-            for item in error_summary:
-                if isinstance(item, dict):
-                    count = item.get("count", item.get("Count"))
-                    if count is not None:
-                        try:
-                            c = int(count)
-                            total_errors += c
-                            code = item.get("code", item.get("Code"))
-                            if code is not None:
-                                metrics[f"request_count_errors_{code}"] = float(c)
-                        except (ValueError, TypeError):
-                            pass
-            if total_errors > 0:
-                metrics["request_count_errors_total"] = float(total_errors)
-        elif isinstance(error_summary, dict):
-            # e.g. {"429": 93617, "404": 3318}
-            total_errors = 0
-            for code, count in error_summary.items():
-                try:
-                    c = int(count)
-                    total_errors += c
-                    metrics[f"request_count_errors_{code}"] = float(c)
-                except (ValueError, TypeError):
-                    pass
-            if total_errors > 0:
-                metrics["request_count_errors_total"] = float(total_errors)
-                    
-    except json.JSONDecodeError as e:
-        print(f"⚠️  Invalid JSON in {json_file}: {e}")
-    except Exception as e:
-        print(f"⚠️  Failed to parse {json_file}: {e}")
-        import traceback
-        traceback.print_exc()
-    
-    if metrics:
-        print(f"✅ Parsed {len(metrics)} metrics from {json_file.name}")
-    else:
-        print(f"⚠️  No metrics extracted from {json_file.name}")
-        # Debug: print structure
-        try:
-            with open(json_file) as f:
-                data = json.load(f)
-                print(f"   File structure keys: {list(data.keys())[:20]}")
-                # Show sample of a metric structure if available
-                for key in ["request_latency", "time_to_first_token", "request_throughput"]:
-                    if key in data and isinstance(data[key], dict):
-                        print(f"   Sample {key} structure: {dict(list(data[key].items())[:5])}")
-                        break
-        except Exception as e:
-            print(f"   Error reading file for debug: {e}")
-    
-    return metrics
-
-
-def main():
-    """Main entry point for the benchmark script."""
-    # Get configuration from environment variables or use defaults
-    # API gateway: ENDPOINT_URL must be BASE URL only (e.g. https://api.hyperbolic.xyz).
-    # AIPerf appends /v1/chat/completions from endpoint-type; if you pass full URL you get double path → 404.
-    # We normalize full URLs to base so both "https://api.hyperbolic.xyz" and "https://api.hyperbolic.xyz/v1/chat/completions" work.
-    model_name = os.getenv("MODEL_NAME", "Qwen/Qwen3-VL-32B-Thinking")
-    endpoint_url = _normalize_endpoint_url(
-        os.getenv(
-            "ENDPOINT_URL",
-            "https://api.hyperbolic.xyz",
-        )
-    )
-    endpoint_type = os.getenv("ENDPOINT_TYPE", "chat")
-    concurrency = int(os.getenv("CONCURRENCY", "10"))
-    request_count = int(os.getenv("REQUEST_COUNT", "100"))
-    streaming = os.getenv("STREAMING", "true").lower() == "true"
-    output_dir = os.getenv("OUTPUT_DIR", "/tmp/aiperf-results")
-    
-    # Auth: API key for api.hyperbolic.xyz, or Cloudflare Access for inference.hyperbolic.xyz
-    api_key = os.getenv("OPENAI_API_KEY") or os.getenv("HYPERBOLIC_API_KEY")
-    cf_access_client_id = os.getenv("CF_ACCESS_CLIENT_ID")
-    cf_access_client_secret = os.getenv("CF_ACCESS_CLIENT_SECRET")
-    
-    # Additional optional parameters
-    timeout = os.getenv("REQUEST_TIMEOUT")
-    output_tokens_mean = os.getenv("OUTPUT_TOKENS_MEAN")
-    benchmark_duration = os.getenv("BENCHMARK_DURATION")
-    benchmark_grace_period = os.getenv("BENCHMARK_GRACE_PERIOD")
-    
-    kwargs = {}
-    if timeout:
-        kwargs["request_timeout_seconds"] = float(timeout)
-    if output_tokens_mean:
-        kwargs["output_tokens_mean"] = int(output_tokens_mean)
-    if benchmark_duration:
-        kwargs["benchmark_duration"] = float(benchmark_duration)
-    if benchmark_grace_period:
-        kwargs["benchmark_grace_period"] = float(benchmark_grace_period)
-    
-    # Run benchmark
-    results = run_benchmark(
-        model_name=model_name,
-        endpoint_url=endpoint_url,
-        endpoint_type=endpoint_type,
-        concurrency=concurrency,
-        request_count=request_count,
-        streaming=streaming,
-        output_dir=output_dir,
-        cf_access_client_id=cf_access_client_id,
-        cf_access_client_secret=cf_access_client_secret,
-        api_key=api_key,
-        **kwargs
-    )
-    
-    # Print results summary
-    print()
-    print("=" * 60)
-    print("Benchmark Summary")
-    print("=" * 60)
-    print(json.dumps(results, indent=2))
-    
-    # Validate benchmark results even if AIPerf exited with code 0
-    # AIPerf may exit with code 0 but fail to produce valid results
-    benchmark_failed = False
-    if results.get("status") != "success":
-        benchmark_failed = True
-        print(f"❌ Benchmark failed with status: {results.get('status')}")
-    else:
-        # Check if valid result files were produced
-        print()
-        print("=" * 60)
-        print("Validating Benchmark Results")
-        print("=" * 60)
-        
-        # Parse metrics to validate results
-        metrics = parse_aiperf_results(output_dir)
-        
-        # Check for required result file
-        result_file = Path(output_dir) / "profile_export_aiperf.json"
-        if not result_file.exists():
-            print(f"⚠️  Required result file not found: {result_file}")
-            benchmark_failed = True
-        elif not metrics:
-            print(f"⚠️  Result file exists but no metrics extracted")
-            print(f"   This may indicate the benchmark didn't complete successfully")
-            benchmark_failed = True
-        else:
-            # Check for critical metrics that should always be present
-            critical_metrics = ["request_latency_avg", "time_to_first_token_avg", "request_count_avg"]
-            missing_critical = [m for m in critical_metrics if m not in metrics]
-            if missing_critical:
-                print(f"⚠️  Missing critical metrics: {missing_critical}")
-                print(f"   This may indicate incomplete benchmark results")
-                # Don't fail on this - some metrics may be optional
-                # But log it for visibility
-    
-    # Send to Datadog if benchmark was successful and has valid results
-    if results.get("status") == "success" and not benchmark_failed:
-        print()
-        print("=" * 60)
-        print("Sending Results to Datadog")
-        print("=" * 60)
-        
-        # Parse metrics (already parsed above, but parse again to ensure we have latest)
-        metrics = parse_aiperf_results(output_dir)
-        
-        if metrics:
-            # Prepare tags
-            base_tags = [
-                f"model:{model_name}",
-                f"endpoint:{endpoint_url}",
-                "benchmark:aiperf",
-                "cluster_name:inference-cluster"
-            ]
-            
-            # Send asynchronously
-            send_metrics_async(
-                metrics=metrics,
-                metric_prefix="inference.benchmark.aiperf",
-                base_tags=base_tags
-            )
-        else:
-            print("⚠️  No metrics to send to Datadog")
-    
-    # Exit with appropriate code
-    if benchmark_failed:
-        print()
-        print("=" * 60)
-        print("❌ Benchmark Failed: Invalid or missing results")
-        print("=" * 60)
-        sys.exit(1)
-    
-    sys.exit(0 if results.get("status") == "success" else 1)
-
-
-if __name__ == "__main__":
-    main()
+#!/usr/bin/env python3
+"""
+AIPerf Benchmark Script for Inference Endpoints
+Runs performance benchmarks against inference endpoints using AIPerf
+"""
+
+import os
+import sys
+import subprocess
+import json
+from pathlib import Path
+from datetime import datetime, timezone
+from typing import Optional, Dict, Any, Iterable, Set
+import re
+
+# Add common directory to path
+sys.path.insert(0, '/scripts/common')
+from datadog_utils import send_metrics_async
+
+
+# Header names whose values are credentials and must never reach stdout, log
+# aggregation, or the returned result payload. Matching is case-insensitive
+# because AIPerf echoes the CLI arguments back verbatim.
+SENSITIVE_HEADER_NAMES = (
+    "authorization",
+    "cf-access-client-secret",
+    "cf-access-client-id",
+    "x-api-key",
+    "api-key",
+    "proxy-authorization",
+)
+
+REDACTED = "***REDACTED***"
+
+
+def _utc_now_iso() -> str:
+    """Return the current UTC time as a timezone-aware ISO-8601 string.
+
+    `datetime.utcnow()` returns a naive datetime and is deprecated as of Python
+    3.12, which makes the emitted timestamps ambiguous once they are ingested by
+    downstream systems. Using an explicit UTC offset keeps them unambiguous.
+    """
+    return datetime.now(timezone.utc).isoformat()
+
+
+def _redact_header_argument(part: str) -> str:
+    """Redact the value of a `Name: value` header argument if the name is sensitive.
+
+    Returns the argument unchanged when it is not a sensitive header, so the
+    logged command line stays useful for debugging.
+    """
+    if ":" not in part:
+        return part
+    name, _, _value = part.partition(":")
+    if name.strip().lower() in SENSITIVE_HEADER_NAMES:
+        return f"{name}: {REDACTED}"
+    return part
+
+
+def _collect_secret_values(*values: Optional[str]) -> Set[str]:
+    """Build the set of literal secret values that must be scrubbed from output.
+
+    Short values are ignored: replacing a 1-3 character string would corrupt
+    unrelated output without providing any protection.
+    """
+    return {value for value in values if value and len(value) >= 4}
+
+
+def _redact_secrets(text: Optional[str], secrets: Iterable[str]) -> str:
+    """Remove credentials from captured process output.
+
+    Two passes are applied:
+      1. Literal replacement of every known secret value. This covers the case
+         where AIPerf echoes the resolved CLI arguments.
+      2. Pattern-based replacement of `Name: value` header pairs and bearer
+         tokens, which covers secrets that are not known to this process (for
+         example values injected by a wrapper or read from a config file).
+    """
+    if not text:
+        return ""
+
+    redacted = text
+    for secret in secrets:
+        redacted = redacted.replace(secret, REDACTED)
+
+    # Scrub any sensitive header that appears as "Name: value" in the output.
+    # The lookahead is anchored immediately after the colon so an already
+    # redacted value cannot be matched again — otherwise the greedy value class
+    # would swallow the remainder of the line on every subsequent pass.
+    header_pattern = re.compile(
+        r"(?i)\b(" + "|".join(re.escape(name) for name in SENSITIVE_HEADER_NAMES) + r")"
+        r"\s*:(?![ \t]*" + re.escape(REDACTED) + r")[ \t]*[^\r\n'\"]+"
+    )
+    redacted = header_pattern.sub(lambda m: f"{m.group(1)}: {REDACTED}", redacted)
+
+    # Scrub bearer tokens that may appear without the header name attached.
+    redacted = re.sub(
+        r"(?i)\bBearer(?![ \t]*" + re.escape(REDACTED) + r")[ \t]+[A-Za-z0-9._\-]{4,}",
+        f"Bearer {REDACTED}",
+        redacted,
+    )
+
+    return redacted
+
+
+def _resolve_subprocess_timeout(
+    benchmark_duration: Optional[int],
+    benchmark_grace_period: Optional[int],
+) -> Optional[float]:
+    """Compute the wall-clock ceiling for the AIPerf child process, in seconds.
+
+    Resolution order:
+      1. `AIPERF_SUBPROCESS_TIMEOUT` if set (`0` or a negative value disables the
+         timeout entirely, preserving the previous unbounded behaviour for
+         operators who rely on it).
+      2. `benchmark_duration` plus `benchmark_grace_period` plus a startup and
+         teardown margin, when a duration-bounded run was requested.
+      3. `AIPERF_DEFAULT_TIMEOUT_SECONDS` (default 4 hours) for request-count
+         runs, whose duration cannot be predicted ahead of time.
+    """
+    override = os.getenv("AIPERF_SUBPROCESS_TIMEOUT")
+    if override:
+        try:
+            value = float(override)
+        except ValueError:
+            print(f"⚠️  Ignoring invalid AIPERF_SUBPROCESS_TIMEOUT={override!r}")
+        else:
+            return None if value <= 0 else value
+
+    # Margin covers AIPerf's own startup/dataset-configuration timeouts plus
+    # artifact export after the measured window closes.
+    startup_and_teardown_margin = 900.0
+
+    if benchmark_duration is not None:
+        return float(benchmark_duration) + float(benchmark_grace_period or 0) + startup_and_teardown_margin
+
+    try:
+        return float(os.getenv("AIPERF_DEFAULT_TIMEOUT_SECONDS", "14400"))
+    except ValueError:
+        return 14400.0
+
+
+def _normalize_endpoint_url(url: str) -> str:
+    """Normalize endpoint URL to base only so AIPerf does not double-append path.
+
+    AIPerf appends the endpoint path (e.g. /v1/chat/completions) to --url. If --url
+    already contains that path, the request goes to /v1/chat/completions/v1/chat/completions → 404.
+    Strip known paths so both "https://api.hyperbolic.xyz" and
+    "https://api.hyperbolic.xyz/v1/chat/completions" result in base URL only.
+    """
+    url = (url or "").rstrip("/")
+    # Strip trailing path if it matches OpenAI-style chat/completions path
+    for suffix in ("/v1/chat/completions", "/v1/completions", "/v1/embeddings"):
+        if url.lower().endswith(suffix):
+            return url[: -len(suffix)].rstrip("/") or url
+    return url
+
+
+def run_benchmark(
+    model_name: str,
+    endpoint_url: str,
+    endpoint_type: str = "chat",
+    concurrency: int = 20,
+    request_count: int = 100,
+    streaming: bool = True,
+    output_dir: str = "/tmp/aiperf-results",
+    benchmark_duration: Optional[int] = None,
+    benchmark_grace_period: Optional[int] = None,
+    request_timeout_seconds: Optional[float] = None,
+    output_tokens_mean: Optional[int] = None,
+    cf_access_client_id: Optional[str] = None,
+    cf_access_client_secret: Optional[str] = None,
+    api_key: Optional[str] = None,
+    **kwargs
+) -> Dict[str, Any]:
+    """
+    Run AIPerf benchmark against an inference endpoint.
+    
+    Args:
+        model_name: Model identifier for AIPerf (must be valid HuggingFace id, e.g. "Qwen/Qwen3-VL-32B-Thinking")
+        endpoint_url: Chat endpoint URL (e.g. https://api.hyperbolic.xyz/v1/chat/completions; model is passed in request body)
+        endpoint_type: Type of endpoint (chat, completions, embeddings)
+        concurrency: Number of concurrent requests
+        request_count: Total number of requests
+        streaming: Enable streaming
+        output_dir: Directory for results
+        output_tokens_mean: Mean number of output tokens per response
+        cf_access_client_id: Cloudflare Access Client ID (optional, for inference.hyperbolic.xyz)
+        cf_access_client_secret: Cloudflare Access Client Secret (optional)
+        api_key: API key for Bearer auth (optional, for api.hyperbolic.xyz)
+        **kwargs: Additional AIPerf arguments
+    
+    Returns:
+        Dictionary with benchmark results and metadata
+    """
+    print("=" * 60)
+    print("AIPerf Benchmark")
+    print("=" * 60)
+    print(f"Model: {model_name}")
+    print(f"Endpoint: {endpoint_url}")
+    print(f"Type: {endpoint_type}")
+    print(f"Concurrency: {concurrency}")
+    if benchmark_duration:
+        print(f"Duration: {benchmark_duration}s")
+        if benchmark_grace_period:
+            print(f"Grace Period: {benchmark_grace_period}s")
+    else:
+        print(f"Request Count: {request_count}")
+    if output_tokens_mean:
+        print(f"Output Tokens Mean: {output_tokens_mean}")
+    print(f"Streaming: {streaming}")
+    print(f"Output Directory: {output_dir}")
+    if api_key:
+        print("Auth: API key (Bearer)")
+    elif cf_access_client_id:
+        # Never print any portion of the credential, not even a prefix: the
+        # client ID is half of a service-token pair and log aggregators retain
+        # this output far longer than the credential rotation window.
+        print("Cloudflare Access: Enabled (service token)")
+    print("=" * 60)
+    print()
+    
+    # Create output directory
+    output_path = Path(output_dir)
+    output_path.mkdir(parents=True, exist_ok=True)
+    
+    # Build AIPerf command
+    cmd = [
+        "aiperf", "profile",
+        "--model", model_name,
+        "--url", endpoint_url,
+        "--endpoint-type", endpoint_type,
+        "--concurrency", str(concurrency),
+        "--output-artifact-dir", output_dir,
+        "--ui-type", "none",  # Disable TUI for non-interactive environments (Kubernetes/containers)
+        "--no-server-metrics",  # Disable server metrics to avoid timeout on unreachable Prometheus endpoints (known issue in v0.4.0+)
+    ]
+    
+    # AIPerf doesn't allow both --request-count and --benchmark-duration together
+    if benchmark_duration is not None:
+        cmd.extend(["--benchmark-duration", str(benchmark_duration)])
+        if benchmark_grace_period is not None:
+            cmd.extend(["--benchmark-grace-period", str(benchmark_grace_period)])
+    else:
+        cmd.extend(["--request-count", str(request_count)])
+    
+    if streaming:
+        cmd.append("--streaming")
+    
+    if request_timeout_seconds is not None:
+        cmd.extend(["--request-timeout-seconds", str(request_timeout_seconds)])
+    
+    if output_tokens_mean is not None:
+        cmd.extend(["--output-tokens-mean", str(output_tokens_mean)])
+        # Send max_tokens in request body (not max_completion_tokens) so backends and gateways apply the limit
+        cmd.append("--use-legacy-max-tokens")
+    
+    # Add auth: API key (Bearer) for api.hyperbolic.xyz, else Cloudflare Access for inference.hyperbolic.xyz
+    if api_key:
+        cmd.extend(["--header", f"Authorization: Bearer {api_key}"])
+    elif cf_access_client_id and cf_access_client_secret:
+        cmd.extend(["--header", f"CF-Access-Client-Id: {cf_access_client_id}"])
+        cmd.extend(["--header", f"CF-Access-Client-Secret: {cf_access_client_secret}"])
+    
+    # Add any additional kwargs
+    for key, value in kwargs.items():
+        if value is not None:
+            flag_name = key.replace('_', '-')
+            cmd.extend([f"--{flag_name}", str(value)])
+    
+    # Redact every sensitive header value in the logged command. The previous
+    # implementation only handled "Authorization: Bearer ...", which left the
+    # Cloudflare Access service-token secret in plain text in pod logs.
+    cmd_safe = [_redact_header_argument(part) for part in cmd]
+    print(f"Running command: {' '.join(cmd_safe)}")
+    print()
+
+    # Every literal secret this process knows about, used to scrub captured
+    # output before it is printed or returned to the caller.
+    secret_values = _collect_secret_values(api_key, cf_access_client_secret, cf_access_client_id)
+    
+    # Prepare environment variables to disable TUI in non-interactive environments
+    env = os.environ.copy()
+    env['TERM'] = 'dumb'  # Disable terminal capabilities
+    env['CI'] = 'true'    # Signal CI/non-interactive environment
+    env['NO_COLOR'] = '1'  # Disable ANSI color codes
+    env['PYTHONUNBUFFERED'] = '1'  # Disable Python output buffering
+    
+    # Set AIPerf timeout environment variables to prevent timeout errors
+    # These timeouts control how long AIPerf waits for services to respond during startup
+    # Increase them for slower environments (e.g., Kubernetes with resource constraints)
+    env.setdefault('AIPERF_SERVICE_PROFILE_CONFIGURE_TIMEOUT', '600.0')  # 10 minutes for configuration
+    env.setdefault('AIPERF_SERVICE_PROFILE_START_TIMEOUT', '300.0')  # 5 minutes for start profiling
+    env.setdefault('AIPERF_DATASET_CONFIGURATION_TIMEOUT', '600.0')  # 10 minutes for dataset config
+    
+    # Bound the wall-clock time of the child process. Without a timeout a hung
+    # AIPerf process keeps the Kubernetes Job alive until the cluster-level
+    # activeDeadlineSeconds fires (or forever, if none is set) and no metrics
+    # are ever exported. When a benchmark duration is configured we derive the
+    # bound from it; otherwise we fall back to a generous default that can be
+    # overridden with AIPERF_SUBPROCESS_TIMEOUT.
+    subprocess_timeout = _resolve_subprocess_timeout(benchmark_duration, benchmark_grace_period)
+    if subprocess_timeout is not None:
+        print(f"Subprocess timeout: {subprocess_timeout:.0f}s")
+
+    try:
+        # Run AIPerf with --ui-type none to disable TUI completely
+        # This is the proper way to run AIPerf in non-interactive environments (Kubernetes/containers)
+        # With --ui-type none, AIPerf should exit cleanly without TUI-related errors
+        result = subprocess.run(
+            cmd,
+            stdin=subprocess.DEVNULL,  # Redirect stdin to prevent any TUI detection
+            stdout=subprocess.PIPE,    # Use PIPE (not TTY) - makes isatty() return False
+            stderr=subprocess.STDOUT,  # Merge stderr into stdout to capture all output
+            text=True,
+            check=True,  # With --ui-type none, AIPerf should exit cleanly
+            env=env,  # Pass environment with TUI-disabling variables
+            timeout=subprocess_timeout,
+        )
+
+        print("✅ Benchmark completed successfully")
+        print(f"Results saved to: {output_dir}")
+
+        # Scrub credentials from stdout before it is printed or returned:
+        # AIPerf echoes the resolved CLI arguments, which include auth headers.
+        stdout_safe = _redact_secrets(result.stdout, secret_values)
+
+        results = {
+            "status": "success",
+            "timestamp": _utc_now_iso(),
+            "model": model_name,
+            "endpoint": endpoint_url,
+            "output_dir": output_dir,
+            "stdout": stdout_safe,
+        }
+
+        # Try to find and parse result files
+        result_files = list(output_path.glob("*.json*"))
+        if result_files:
+            results["result_files"] = [str(f) for f in result_files]
+
+        return results
+
+    except subprocess.TimeoutExpired as e:
+        # Treated as a failure rather than a crash so that partial artifacts are
+        # still reported and the exit path stays identical to other failures.
+        print(f"❌ Benchmark timed out after {e.timeout:.0f}s")
+        result_files = list(output_path.glob("*.json*"))
+        return {
+            "status": "error",
+            "timestamp": _utc_now_iso(),
+            "error": f"AIPerf timed out after {e.timeout:.0f}s",
+            "exit_code": None,
+            "stdout": _redact_secrets(
+                e.stdout.decode(errors="replace") if isinstance(e.stdout, bytes) else e.stdout,
+                secret_values,
+            ),
+            "result_files": [str(f) for f in result_files] if result_files else None,
+        }
+    except subprocess.CalledProcessError as e:
+        print(f"❌ Benchmark failed with exit code {e.returncode}")
+
+        # Check if result files exist despite the error (partial success)
+        result_files = list(output_path.glob("*.json*"))
+        if result_files:
+            print(f"⚠️  Warning: Exit code {e.returncode}, but result files found:")
+            for f in result_files:
+                print(f"   - {f}")
+            print("   This may indicate a non-fatal error (e.g., timeout during cleanup)")
+
+        # Redact before previewing: the failure output is the most likely place
+        # for a credential to surface, because AIPerf prints the failing request.
+        stdout_safe = _redact_secrets(e.stdout, secret_values)
+        stderr_safe = _redact_secrets(e.stderr, secret_values)
+
+        # Show error output (truncated for readability)
+        print(f"\nStdout preview (first 1000 chars):\n{stdout_safe[:1000] or '(empty)'}")
+        if stderr_safe and stderr_safe != stdout_safe:
+            print(f"\nStderr preview (first 1000 chars):\n{stderr_safe[:1000]}")
+
+        # Check for common error patterns
+        error_msg = stderr_safe or stdout_safe or "Unknown error"
+        if "TimeoutError" in error_msg or "timeout" in error_msg.lower():
+            print("\n💡 Tip: If you see timeout errors, try increasing:")
+            print("   - AIPERF_SERVICE_PROFILE_START_TIMEOUT")
+            print("   - AIPERF_SERVICE_PROFILE_CONFIGURE_TIMEOUT")
+            print("   - AIPERF_DATASET_CONFIGURATION_TIMEOUT")
+
+        return {
+            "status": "error",
+            "timestamp": _utc_now_iso(),
+            "error": error_msg,
+            "exit_code": e.returncode,
+            "stdout": stdout_safe,
+            "stderr": stderr_safe,
+            "result_files": [str(f) for f in result_files] if result_files else None,
+        }
+    except FileNotFoundError:
+        print("❌ AIPerf not found. Please install it with: pip install aiperf")
+        return {
+            "status": "error",
+            "timestamp": _utc_now_iso(),
+            "error": "AIPerf not installed",
+        }
+
+
+def parse_aiperf_results(result_dir: str) -> Dict[str, float]:
+    """
+    Parse AIPerf JSON result files.
+    
+    AIPerf exports metrics in profile_export_aiperf.json with structure:
+    {
+        "schema_version": "1.0",
+        "aiperf_version": "...",
+        "benchmark_id": "...",
+        "request_latency": {
+            "unit": "ms",
+            "avg": 762.60,
+            "p50": 752.52,
+            "p95": 790.74,
+            "p99": 1013.11,
+            "min": 705.02,
+            "max": 1725.33,
+            "std": 44.68,
+            ...
+        },
+        "time_to_first_token": { ... },
+        ...
+    }
+    
+    Each metric field is a JsonMetricResult object with: unit, avg, p1, p5, p10, p25, p50, p75, p90, p95, p99, min, max, std
+    
+    Args:
+        result_dir: Directory containing JSON result files
+    
+    Returns:
+        Dictionary of metric_name_stat -> value (e.g., "request_latency_avg", "time_to_first_token_p99")
+    """
+    result_path = Path(result_dir)
+    metrics = {}
+    
+    # Look specifically for profile_export_aiperf.json (the aggregated stats file)
+    json_file = result_path / "profile_export_aiperf.json"
+    
+    if not json_file.exists():
+        print(f"⚠️  No profile_export_aiperf.json found in {result_dir}")
+        return metrics
+    
+    try:
+        with open(json_file) as f:
+            data = json.load(f)
+        
+        if not isinstance(data, dict):
+            print(f"⚠️  Expected dict in {json_file}, got {type(data)}")
+            return metrics
+        
+        # AIPerf JsonExportData structure: metric fields are top-level keys containing JsonMetricResult objects
+        # JsonMetricResult has: unit, avg, p1, p5, p10, p25, p50, p75, p90, p95, p99, min, max, std
+        
+        # Known non-metric fields to skip
+        non_metric_fields = {
+            "schema_version", "aiperf_version", "benchmark_id", "input_config",
+            "was_cancelled", "error_summary", "start_time", "end_time", "telemetry_data"
+        }
+        
+        # All possible stat fields in JsonMetricResult
+        stat_fields = ["avg", "p1", "p5", "p10", "p25", "p50", "p75", "p90", "p95", "p99", "min", "max", "std"]
+        
+        # Extract metrics from all top-level keys that are JsonMetricResult objects
+        for key, value in data.items():
+            # Skip known non-metric fields
+            if key in non_metric_fields:
+                continue
+            
+            # Check if this is a JsonMetricResult (has "unit" field)
+            if isinstance(value, dict) and "unit" in value:
+                # Extract all available stats from this JsonMetricResult
+                for stat in stat_fields:
+                    if stat in value and value[stat] is not None:
+                        try:
+                            metrics[f"{key}_{stat}"] = float(value[stat])
+                        except (ValueError, TypeError):
+                            pass
+        
+        # Extract error_summary so we send request/error counts even when 0 successful requests
+        # (dashboard can then show "last run had X errors" instead of empty)
+        error_summary = data.get("error_summary")
+        if isinstance(error_summary, list):
+            total_errors = 0
+            for item in error_summary:
+                if isinstance(item, dict):
+                    count = item.get("count", item.get("Count"))
+                    if count is not None:
+                        try:
+                            c = int(count)
+                            total_errors += c
+                            code = item.get("code", item.get("Code"))
+                            if code is not None:
+                                metrics[f"request_count_errors_{code}"] = float(c)
+                        except (ValueError, TypeError):
+                            pass
+            if total_errors > 0:
+                metrics["request_count_errors_total"] = float(total_errors)
+        elif isinstance(error_summary, dict):
+            # e.g. {"429": 93617, "404": 3318}
+            total_errors = 0
+            for code, count in error_summary.items():
+                try:
+                    c = int(count)
+                    total_errors += c
+                    metrics[f"request_count_errors_{code}"] = float(c)
+                except (ValueError, TypeError):
+                    pass
+            if total_errors > 0:
+                metrics["request_count_errors_total"] = float(total_errors)
+                    
+    except json.JSONDecodeError as e:
+        print(f"⚠️  Invalid JSON in {json_file}: {e}")
+    except Exception as e:
+        print(f"⚠️  Failed to parse {json_file}: {e}")
+        import traceback
+        traceback.print_exc()
+    
+    if metrics:
+        print(f"✅ Parsed {len(metrics)} metrics from {json_file.name}")
+    else:
+        print(f"⚠️  No metrics extracted from {json_file.name}")
+        # Debug: print structure
+        try:
+            with open(json_file) as f:
+                data = json.load(f)
+                print(f"   File structure keys: {list(data.keys())[:20]}")
+                # Show sample of a metric structure if available
+                for key in ["request_latency", "time_to_first_token", "request_throughput"]:
+                    if key in data and isinstance(data[key], dict):
+                        print(f"   Sample {key} structure: {dict(list(data[key].items())[:5])}")
+                        break
+        except Exception as e:
+            print(f"   Error reading file for debug: {e}")
+    
+    return metrics
+
+
+def main():
+    """Main entry point for the benchmark script."""
+    # Get configuration from environment variables or use defaults
+    # API gateway: ENDPOINT_URL must be BASE URL only (e.g. https://api.hyperbolic.xyz).
+    # AIPerf appends /v1/chat/completions from endpoint-type; if you pass full URL you get double path → 404.
+    # We normalize full URLs to base so both "https://api.hyperbolic.xyz" and "https://api.hyperbolic.xyz/v1/chat/completions" work.
+    model_name = os.getenv("MODEL_NAME", "Qwen/Qwen3-VL-32B-Thinking")
+    endpoint_url = _normalize_endpoint_url(
+        os.getenv(
+            "ENDPOINT_URL",
+            "https://api.hyperbolic.xyz",
+        )
+    )
+    endpoint_type = os.getenv("ENDPOINT_TYPE", "chat")
+    concurrency = int(os.getenv("CONCURRENCY", "10"))
+    request_count = int(os.getenv("REQUEST_COUNT", "100"))
+    streaming = os.getenv("STREAMING", "true").lower() == "true"
+    output_dir = os.getenv("OUTPUT_DIR", "/tmp/aiperf-results")
+    
+    # Auth: API key for api.hyperbolic.xyz, or Cloudflare Access for inference.hyperbolic.xyz
+    api_key = os.getenv("OPENAI_API_KEY") or os.getenv("HYPERBOLIC_API_KEY")
+    cf_access_client_id = os.getenv("CF_ACCESS_CLIENT_ID")
+    cf_access_client_secret = os.getenv("CF_ACCESS_CLIENT_SECRET")
+    
+    # Additional optional parameters
+    timeout = os.getenv("REQUEST_TIMEOUT")
+    output_tokens_mean = os.getenv("OUTPUT_TOKENS_MEAN")
+    benchmark_duration = os.getenv("BENCHMARK_DURATION")
+    benchmark_grace_period = os.getenv("BENCHMARK_GRACE_PERIOD")
+    
+    kwargs = {}
+    if timeout:
+        # AIPerf accepts a fractional request timeout, so float is correct here.
+        kwargs["request_timeout_seconds"] = float(timeout)
+    if output_tokens_mean:
+        kwargs["output_tokens_mean"] = int(output_tokens_mean)
+    if benchmark_duration:
+        # These two flags are whole seconds. Casting to float produced values
+        # like "--benchmark-duration 480.0", which AIPerf rejects as invalid
+        # integers, so the run failed before a single request was sent.
+        kwargs["benchmark_duration"] = int(float(benchmark_duration))
+    if benchmark_grace_period:
+        kwargs["benchmark_grace_period"] = int(float(benchmark_grace_period))
+    
+    # Run benchmark
+    results = run_benchmark(
+        model_name=model_name,
+        endpoint_url=endpoint_url,
+        endpoint_type=endpoint_type,
+        concurrency=concurrency,
+        request_count=request_count,
+        streaming=streaming,
+        output_dir=output_dir,
+        cf_access_client_id=cf_access_client_id,
+        cf_access_client_secret=cf_access_client_secret,
+        api_key=api_key,
+        **kwargs
+    )
+    
+    # Print results summary. `stdout` is excluded: it was already streamed to the
+    # log once by AIPerf, it can be megabytes long, and echoing captured process
+    # output a second time only increases the chance of leaking something.
+    print()
+    print("=" * 60)
+    print("Benchmark Summary")
+    print("=" * 60)
+    summary = {key: value for key, value in results.items() if key not in ("stdout", "stderr")}
+    print(json.dumps(summary, indent=2))
+    
+    # Validate benchmark results even if AIPerf exited with code 0
+    # AIPerf may exit with code 0 but fail to produce valid results
+    benchmark_failed = False
+    metrics: Dict[str, float] = {}
+    if results.get("status") != "success":
+        benchmark_failed = True
+        print(f"❌ Benchmark failed with status: {results.get('status')}")
+    else:
+        # Check if valid result files were produced
+        print()
+        print("=" * 60)
+        print("Validating Benchmark Results")
+        print("=" * 60)
+
+        # Parse metrics to validate results
+        metrics = parse_aiperf_results(output_dir)
+
+        # Check for required result file
+        result_file = Path(output_dir) / "profile_export_aiperf.json"
+        if not result_file.exists():
+            print(f"⚠️  Required result file not found: {result_file}")
+            benchmark_failed = True
+        elif not metrics:
+            print("⚠️  Result file exists but no metrics extracted")
+            print("   This may indicate the benchmark didn't complete successfully")
+            benchmark_failed = True
+        else:
+            # Check for critical metrics that should always be present
+            critical_metrics = ["request_latency_avg", "time_to_first_token_avg", "request_count_avg"]
+            missing_critical = [m for m in critical_metrics if m not in metrics]
+            if missing_critical:
+                print(f"⚠️  Missing critical metrics: {missing_critical}")
+                print("   This may indicate incomplete benchmark results")
+                # Don't fail on this - some metrics may be optional
+                # But log it for visibility
+
+    # Send to Datadog if benchmark was successful and has valid results
+    if results.get("status") == "success" and not benchmark_failed:
+        print()
+        print("=" * 60)
+        print("Sending Results to Datadog")
+        print("=" * 60)
+
+        # Reuse the metrics parsed during validation instead of re-reading and
+        # re-parsing the export file, which cannot have changed in between.
+        if metrics:
+            # Prepare tags
+            base_tags = [
+                f"model:{model_name}",
+                f"endpoint:{endpoint_url}",
+                "benchmark:aiperf",
+                "cluster_name:inference-cluster"
+            ]
+            
+            # Send asynchronously
+            send_metrics_async(
+                metrics=metrics,
+                metric_prefix="inference.benchmark.aiperf",
+                base_tags=base_tags
+            )
+        else:
+            print("⚠️  No metrics to send to Datadog")
+    
+    # Exit with appropriate code
+    if benchmark_failed:
+        print()
+        print("=" * 60)
+        print("❌ Benchmark Failed: Invalid or missing results")
+        print("=" * 60)
+        sys.exit(1)
+    
+    sys.exit(0 if results.get("status") == "success" else 1)
+
+
+if __name__ == "__main__":
+    main()
diff --git a/aiperf/create-api-key-secret.sh b/aiperf/create-api-key-secret.sh
index 66c7122..ad92575 100755
--- a/aiperf/create-api-key-secret.sh
+++ b/aiperf/create-api-key-secret.sh
@@ -1,42 +1,45 @@
-#!/bin/bash
-# Create Kubernetes secret for api.hyperbolic.xyz API key (benchmarks).
-# Usage: OPENAI_API_KEY=your-key ./create-api-key-secret.sh [NAMESPACE]
-#   or:  HYPERBOLIC_API_KEY=your-key ./create-api-key-secret.sh [NAMESPACE]
-# Never commit the key to the repo; use env var or paste when prompted.
-
-set -e
-
-NAMESPACE="${1:-inference-benchmark}"
-SECRET_NAME="hyperbolic-api-key"
-
-GREEN='\033[0;32m'
-YELLOW='\033[1;33m'
-RED='\033[0;31m'
-BLUE='\033[0;34m'
-NC='\033[0m'
-
-API_KEY="${OPENAI_API_KEY:-${HYPERBOLIC_API_KEY:-}}"
-if [ -z "$API_KEY" ]; then
-    echo -e "${YELLOW}OPENAI_API_KEY / HYPERBOLIC_API_KEY not set. Paste API key (then Ctrl+D):${NC}"
-    API_KEY=$(cat)
-fi
-if [ -z "$API_KEY" ]; then
-    echo -e "${RED}❌ No API key provided${NC}"
-    echo "Usage: OPENAI_API_KEY=your-key $0 [NAMESPACE]"
-    exit 1
-fi
-
-echo -e "${BLUE}Creating secret ${SECRET_NAME} in namespace ${NAMESPACE}...${NC}"
-
-if ! kubectl get namespace "$NAMESPACE" &>/dev/null; then
-    echo -e "${YELLOW}Namespace '$NAMESPACE' does not exist. Creating it...${NC}"
-    kubectl create namespace "$NAMESPACE"
-fi
-
-kubectl create secret generic "$SECRET_NAME" \
-    --from-literal=api-key="$API_KEY" \
-    -n "$NAMESPACE" \
-    --dry-run=client -o yaml | kubectl apply -f -
-
-echo -e "${GREEN}✅ Secret ${SECRET_NAME} created/updated in ${NAMESPACE}${NC}"
-echo "   CronJob will use it for api.hyperbolic.xyz auth."
+#!/usr/bin/env bash
+# Create Kubernetes secret for api.hyperbolic.xyz API key (benchmarks).
+# Usage: OPENAI_API_KEY=your-key ./create-api-key-secret.sh [NAMESPACE]
+#   or:  HYPERBOLIC_API_KEY=your-key ./create-api-key-secret.sh [NAMESPACE]
+# Never commit the key to the repo; use env var or paste when prompted.
+
+# `-u` turns a typo in a variable name into an error rather than an empty secret
+# value, and `pipefail` propagates a failure from the `kubectl create` side of the
+# pipe below, which `set -e` alone does not catch.
+set -euo pipefail
+
+NAMESPACE="${1:-inference-benchmark}"
+SECRET_NAME="hyperbolic-api-key"
+
+GREEN='\033[0;32m'
+YELLOW='\033[1;33m'
+RED='\033[0;31m'
+BLUE='\033[0;34m'
+NC='\033[0m'
+
+API_KEY="${OPENAI_API_KEY:-${HYPERBOLIC_API_KEY:-}}"
+if [ -z "$API_KEY" ]; then
+    echo -e "${YELLOW}OPENAI_API_KEY / HYPERBOLIC_API_KEY not set. Paste API key (then Ctrl+D):${NC}"
+    API_KEY=$(cat)
+fi
+if [ -z "$API_KEY" ]; then
+    echo -e "${RED}❌ No API key provided${NC}"
+    echo "Usage: OPENAI_API_KEY=your-key $0 [NAMESPACE]"
+    exit 1
+fi
+
+echo -e "${BLUE}Creating secret ${SECRET_NAME} in namespace ${NAMESPACE}...${NC}"
+
+if ! kubectl get namespace "$NAMESPACE" &>/dev/null; then
+    echo -e "${YELLOW}Namespace '$NAMESPACE' does not exist. Creating it...${NC}"
+    kubectl create namespace "$NAMESPACE"
+fi
+
+kubectl create secret generic "$SECRET_NAME" \
+    --from-literal=api-key="$API_KEY" \
+    -n "$NAMESPACE" \
+    --dry-run=client -o yaml | kubectl apply -f -
+
+echo -e "${GREEN}✅ Secret ${SECRET_NAME} created/updated in ${NAMESPACE}${NC}"
+echo "   CronJob will use it for api.hyperbolic.xyz auth."
diff --git a/aiperf/create-secret.sh b/aiperf/create-secret.sh
index a0be588..2a66639 100755
--- a/aiperf/create-secret.sh
+++ b/aiperf/create-secret.sh
@@ -1,100 +1,113 @@
-#!/bin/bash
-# Create Kubernetes secret for Cloudflare Access credentials
-# Usage: ./create-secret.sh [CLIENT_ID] [CLIENT_SECRET] [NAMESPACE]
+#!/usr/bin/env bash
+# Create the Kubernetes secret holding Cloudflare Access credentials.
+#
+# Usage:
+#   CLIENT_ID=... CLIENT_SECRET=... ./create-secret.sh [NAMESPACE]
+#   ./create-secret.sh [NAMESPACE]        # prompts for both values
+#
+# Credentials are read from the environment or prompted for. They are
+# deliberately NOT accepted as command-line arguments: every process on the host
+# can read another process's argv (`ps auxww`), and the shell records the command
+# — including the secret — in its history file.
 
-set -e
+# `-u` makes an unset variable a fatal error instead of an empty string (the old
+# script would happily create a secret with an empty value), and `pipefail` means
+# a failure in `kubectl create ... | kubectl apply` is not masked by the exit
+# status of the last stage.
+set -euo pipefail
 
-# Default values
-NAMESPACE="${3:-inference-benchmark}"
+# The old signature was `create-secret.sh CLIENT_ID CLIENT_SECRET [NAMESPACE]`.
+# Now that the namespace is the first argument, silently accepting three
+# arguments would create a namespace named after a client ID, so refuse instead
+# and say what changed.
+if [ "$#" -gt 1 ]; then
+    echo "Error: credentials are no longer passed as arguments (they leak via 'ps' and shell history)." >&2
+    echo "Usage: CLIENT_ID=... CLIENT_SECRET=... $0 [NAMESPACE]" >&2
+    exit 2
+fi
+
+NAMESPACE="${1:-inference-benchmark}"
 SECRET_NAME="cloudflare-access-credentials"
+CREDENTIALS_FILE="${CLOUDFLARE_CREDENTIALS_FILE:-cloudflare-access-credentials.env}"
 
-# Colors for output
 GREEN='\033[0;32m'
 YELLOW='\033[1;33m'
 RED='\033[0;31m'
 BLUE='\033[0;34m'
 NC='\033[0m' # No Color
 
-echo -e "${BLUE}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}"
 echo -e "${BLUE}Cloudflare Access Credentials Secret Setup${NC}"
-echo -e "${BLUE}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}"
-echo ""
 
-# Get credentials
-if [ -n "$1" ] && [ -n "$2" ]; then
-    CLIENT_ID="$1"
-    CLIENT_SECRET="$2"
-    echo -e "${GREEN}✅ Using provided credentials${NC}"
-elif [ -f "cloudflare-access-credentials.txt" ]; then
-    echo -e "${BLUE}📋 Loading credentials from cloudflare-access-credentials.txt${NC}"
-    source cloudflare-access-credentials.txt
-    CLIENT_ID="$CLIENT_ID"
-    CLIENT_SECRET="$CLIENT_SECRET"
-elif [ -f "../../cloudflare-access-credentials.txt" ]; then
-    echo -e "${BLUE}📋 Loading credentials from ../../cloudflare-access-credentials.txt${NC}"
-    source ../../cloudflare-access-credentials.txt
-    CLIENT_ID="$CLIENT_ID"
-    CLIENT_SECRET="$CLIENT_SECRET"
-else
-    echo -e "${YELLOW}⚠️  No credentials provided${NC}"
-    echo ""
-    echo "Usage:"
-    echo "  $0 CLIENT_ID CLIENT_SECRET [NAMESPACE]"
-    echo ""
-    echo "Or set environment variables:"
-    echo "  export CLIENT_ID='your-client-id'"
-    echo "  export CLIENT_SECRET='your-client-secret'"
-    echo "  $0"
+CLIENT_ID="${CLIENT_ID:-}"
+CLIENT_SECRET="${CLIENT_SECRET:-}"
+
+# A credentials file is still supported, but it is *parsed* rather than sourced.
+# `source` executes the file as shell script, so anything that could write to it
+# — or a stray backtick in a pasted value — ran with the privileges of whoever
+# ran this script. Only `KEY=value` lines for the two expected keys are read.
+if [ -z "$CLIENT_ID" ] || [ -z "$CLIENT_SECRET" ]; then
+    for candidate in "$CREDENTIALS_FILE" "../../$CREDENTIALS_FILE" \
+        "cloudflare-access-credentials.txt" "../../cloudflare-access-credentials.txt"; do
+        if [ -f "$candidate" ]; then
+            echo -e "${BLUE}Loading credentials from ${candidate}${NC}"
+            while IFS='=' read -r key value; do
+                case "$key" in
+                    CLIENT_ID) [ -z "$CLIENT_ID" ] && CLIENT_ID="$value" ;;
+                    CLIENT_SECRET) [ -z "$CLIENT_SECRET" ] && CLIENT_SECRET="$value" ;;
+                esac
+            done < "$candidate"
+            break
+        fi
+    done
+fi
+
+# `read -s` keeps the value off the screen and out of scrollback.
+if [ -z "$CLIENT_ID" ]; then
+    read -r -p "Cloudflare Access Client ID: " CLIENT_ID
+fi
+if [ -z "$CLIENT_SECRET" ]; then
+    read -r -s -p "Cloudflare Access Client Secret: " CLIENT_SECRET
     echo ""
-    echo "Or create cloudflare-access-credentials.txt with:"
-    echo "  CLIENT_ID=your-client-id"
-    echo "  CLIENT_SECRET=your-client-secret"
-    exit 1
 fi
 
-# Validate credentials
 if [ -z "$CLIENT_ID" ] || [ -z "$CLIENT_SECRET" ]; then
-    echo -e "${RED}❌ Error: CLIENT_ID and CLIENT_SECRET are required${NC}"
+    echo -e "${RED}Error: CLIENT_ID and CLIENT_SECRET are both required${NC}" >&2
     exit 1
 fi
 
-echo -e "${BLUE}📦 Creating Kubernetes secret...${NC}"
-echo -e "  Namespace: ${YELLOW}$NAMESPACE${NC}"
-echo -e "  Secret name: ${YELLOW}$SECRET_NAME${NC}"
-echo -e "  Client ID: ${YELLOW}${CLIENT_ID:0:20}...${NC}"
-echo ""
+echo -e "${BLUE}Creating Kubernetes secret...${NC}"
+echo -e "  Namespace:   ${YELLOW}${NAMESPACE}${NC}"
+echo -e "  Secret name: ${YELLOW}${SECRET_NAME}${NC}"
+# The client ID's first 20 characters used to be echoed here. A partial
+# credential is still a credential once it reaches a shared terminal or CI log,
+# and it narrows a brute-force search considerably, so nothing is printed.
 
-# Check if namespace exists
-if ! kubectl get namespace "$NAMESPACE" &>/dev/null; then
-    echo -e "${YELLOW}⚠️  Namespace '$NAMESPACE' does not exist. Creating it...${NC}"
+if ! kubectl get namespace "$NAMESPACE" >/dev/null 2>&1; then
+    echo -e "${YELLOW}Namespace '${NAMESPACE}' does not exist. Creating it...${NC}"
     kubectl create namespace "$NAMESPACE"
-    echo -e "${GREEN}✅ Namespace created${NC}"
 fi
 
-# Delete existing secret if it exists
-if kubectl get secret "$SECRET_NAME" -n "$NAMESPACE" &>/dev/null; then
-    echo -e "${YELLOW}⚠️  Secret '$SECRET_NAME' already exists. Updating it...${NC}"
-    kubectl delete secret "$SECRET_NAME" -n "$NAMESPACE"
-fi
-
-# Create the secret
+# The previous version deleted the secret and then created it again. Between the
+# two calls the secret did not exist, so any pod that started (or restarted) in
+# that window came up without credentials; and if the create failed, the cluster
+# was left with no secret at all. `apply` from a client-side manifest is a single
+# atomic update and is safe to re-run.
 kubectl create secret generic "$SECRET_NAME" \
     --from-literal=client-id="$CLIENT_ID" \
     --from-literal=client-secret="$CLIENT_SECRET" \
-    -n "$NAMESPACE"
+    -n "$NAMESPACE" \
+    --dry-run=client -o yaml | kubectl apply -f -
 
 echo ""
-echo -e "${GREEN}✅ Secret created successfully!${NC}"
-echo ""
-echo -e "${BLUE}📋 Verification:${NC}"
-kubectl get secret "$SECRET_NAME" -n "$NAMESPACE" -o jsonpath='{.data.client-id}' | base64 -d | head -c 20 && echo "..."
+echo -e "${GREEN}Secret ${SECRET_NAME} created/updated in ${NAMESPACE}${NC}"
 echo ""
-echo -e "${GREEN}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}"
-echo -e "${GREEN}✅ Setup Complete!${NC}"
-echo -e "${GREEN}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}"
+# Verification confirms the keys exist without decoding their values; the old
+# `base64 -d | head -c 20` printed the beginning of the real client ID.
+echo -e "${BLUE}Keys present in the secret:${NC}"
+kubectl get secret "$SECRET_NAME" -n "$NAMESPACE" -o jsonpath='{range $k, $v := .data}{$k}{"\n"}{end}'
 echo ""
-echo -e "${BLUE}📝 Next Steps:${NC}"
-echo "  1. The CronJob will automatically use this secret"
-echo "  2. Verify with: kubectl get secret $SECRET_NAME -n $NAMESPACE"
-echo "  3. Check CronJob logs: kubectl logs -n $NAMESPACE -l app=aiperf --tail=50"
+echo -e "${BLUE}Next steps:${NC}"
+echo "  1. The CronJob picks the secret up automatically on its next run."
+echo "  2. Inspect it with: kubectl get secret ${SECRET_NAME} -n ${NAMESPACE}"
+echo "  3. Check CronJob logs: kubectl logs -n ${NAMESPACE} -l app=aiperf --tail=50"
 echo ""
diff --git a/aiperf/cronjob.yaml b/aiperf/cronjob.yaml
index 2c7fb1a..981b8f6 100644
--- a/aiperf/cronjob.yaml
+++ b/aiperf/cronjob.yaml
@@ -1,231 +1,265 @@
-# Two CronJobs run sequentially (staggered) so both endpoints are benchmarked without
-# doubling load. api.hyperbolic.xyz at :00,:20,:40; inference.hyperbolic.ai at :10,:30,:50.
-#
-# Workload is identical for both (same model, concurrency, duration, output tokens, timeout,
-# streaming). Only ENDPOINT_URL and auth differ (api = Bearer key, inference = Cloudflare Access).
-# This keeps endpoint performance comparison in Datadog apples-to-apples.
----
-apiVersion: batch/v1
-kind: CronJob
-metadata:
-  name: aiperf-benchmark-api
-  namespace: inference-benchmark
-  labels:
-    app: aiperf
-    component: benchmark
-    endpoint: api.hyperbolic.xyz
-spec:
-  suspend: true  # set to false to resume benchmarks
-  schedule: "0,20,40 * * * *"
-  concurrencyPolicy: Forbid
-  successfulJobsHistoryLimit: 5
-  failedJobsHistoryLimit: 3
-  jobTemplate:
-    spec:
-      template:
-        metadata:
-          labels:
-            app: aiperf
-            component: benchmark
-        spec:
-          restartPolicy: OnFailure
-          imagePullSecrets:
-            - name: ghcr-image-pull-secret
-          containers:
-            - name: aiperf
-              image: ghcr.io/hyperboliclabs/aiperf:latest
-              imagePullPolicy: Always
-              command:
-                - /bin/sh
-                - -c
-                - |
-                  set -e
-                  echo "=== Running AIPerf Benchmark (api.hyperbolic.xyz) ==="
-                  python3 /scripts/benchmark.py
-                  echo "=== Benchmark Complete ==="
-                  ls -lh /tmp/results/ || echo "No results directory"
-              env:
-                - name: MODEL_NAME
-                  value: "Qwen/Qwen3-VL-32B-Thinking"
-                - name: ENDPOINT_URL
-                  value: "https://api.hyperbolic.xyz"
-                - name: ENDPOINT_TYPE
-                  value: "chat"
-                - name: OPENAI_API_KEY
-                  valueFrom:
-                    secretKeyRef:
-                      name: hyperbolic-api-key
-                      key: api-key
-                      optional: true
-                - name: CF_ACCESS_CLIENT_ID
-                  valueFrom:
-                    secretKeyRef:
-                      name: cloudflare-access-credentials
-                      key: client-id
-                      optional: true
-                - name: CF_ACCESS_CLIENT_SECRET
-                  valueFrom:
-                    secretKeyRef:
-                      name: cloudflare-access-credentials
-                      key: client-secret
-                      optional: true
-                - name: CONCURRENCY
-                  value: "10"
-                - name: BENCHMARK_DURATION
-                  value: "480"
-                - name: BENCHMARK_GRACE_PERIOD
-                  value: "30"
-                - name: REQUEST_TIMEOUT
-                  value: "60"
-                - name: OUTPUT_TOKENS_MEAN
-                  value: "50"
-                - name: REQUEST_COUNT
-                  value: "10000"
-                - name: STREAMING
-                  value: "true"
-                - name: OUTPUT_DIR
-                  value: "/tmp/results"
-                - name: TERM
-                  value: "dumb"
-                - name: CI
-                  value: "true"
-                - name: NO_COLOR
-                  value: "1"
-                - name: PYTHONUNBUFFERED
-                  value: "1"
-                - name: AIPERF_SERVICE_PROFILE_CONFIGURE_TIMEOUT
-                  value: "600.0"
-                - name: AIPERF_SERVICE_PROFILE_START_TIMEOUT
-                  value: "300.0"
-                - name: AIPERF_DATASET_CONFIGURATION_TIMEOUT
-                  value: "600.0"
-                - name: DD_API_KEY
-                  valueFrom:
-                    secretKeyRef:
-                      name: datadog-api-key
-                      key: api-key
-                      optional: true
-                - name: DD_SITE
-                  value: "us5.datadoghq.com"
-              volumeMounts:
-                - name: results
-                  mountPath: /tmp/results
-              resources:
-                requests:
-                  memory: "2Gi"
-                  cpu: "200m"
-                limits:
-                  memory: "4Gi"
-                  cpu: "1000m"
-          volumes:
-            - name: results
-              emptyDir: {}
----
-apiVersion: batch/v1
-kind: CronJob
-metadata:
-  name: aiperf-benchmark-inference
-  namespace: inference-benchmark
-  labels:
-    app: aiperf
-    component: benchmark
-    endpoint: inference.hyperbolic.ai
-spec:
-  suspend: true  # set to false to resume benchmarks
-  schedule: "10,30,50 * * * *"
-  concurrencyPolicy: Forbid
-  successfulJobsHistoryLimit: 5
-  failedJobsHistoryLimit: 3
-  jobTemplate:
-    spec:
-      template:
-        metadata:
-          labels:
-            app: aiperf
-            component: benchmark
-        spec:
-          restartPolicy: OnFailure
-          imagePullSecrets:
-            - name: ghcr-image-pull-secret
-          containers:
-            - name: aiperf
-              image: ghcr.io/hyperboliclabs/aiperf:latest
-              imagePullPolicy: Always
-              command:
-                - /bin/sh
-                - -c
-                - |
-                  set -e
-                  echo "=== Running AIPerf Benchmark (inference.hyperbolic.ai) ==="
-                  python3 /scripts/benchmark.py
-                  echo "=== Benchmark Complete ==="
-                  ls -lh /tmp/results/ || echo "No results directory"
-              env:
-                - name: MODEL_NAME
-                  value: "Qwen/Qwen3-VL-32B-Thinking"
-                - name: ENDPOINT_URL
-                  value: "https://inference.hyperbolic.ai"
-                - name: ENDPOINT_TYPE
-                  value: "chat"
-                # inference.hyperbolic.ai uses Cloudflare Access (no OPENAI_API_KEY)
-                - name: CF_ACCESS_CLIENT_ID
-                  valueFrom:
-                    secretKeyRef:
-                      name: cloudflare-access-credentials
-                      key: client-id
-                - name: CF_ACCESS_CLIENT_SECRET
-                  valueFrom:
-                    secretKeyRef:
-                      name: cloudflare-access-credentials
-                      key: client-secret
-                - name: CONCURRENCY
-                  value: "10"
-                - name: BENCHMARK_DURATION
-                  value: "480"
-                - name: BENCHMARK_GRACE_PERIOD
-                  value: "30"
-                - name: REQUEST_TIMEOUT
-                  value: "60"
-                - name: OUTPUT_TOKENS_MEAN
-                  value: "50"
-                - name: REQUEST_COUNT
-                  value: "10000"
-                - name: STREAMING
-                  value: "true"
-                - name: OUTPUT_DIR
-                  value: "/tmp/results"
-                - name: TERM
-                  value: "dumb"
-                - name: CI
-                  value: "true"
-                - name: NO_COLOR
-                  value: "1"
-                - name: PYTHONUNBUFFERED
-                  value: "1"
-                - name: AIPERF_SERVICE_PROFILE_CONFIGURE_TIMEOUT
-                  value: "600.0"
-                - name: AIPERF_SERVICE_PROFILE_START_TIMEOUT
-                  value: "300.0"
-                - name: AIPERF_DATASET_CONFIGURATION_TIMEOUT
-                  value: "600.0"
-                - name: DD_API_KEY
-                  valueFrom:
-                    secretKeyRef:
-                      name: datadog-api-key
-                      key: api-key
-                      optional: true
-                - name: DD_SITE
-                  value: "us5.datadoghq.com"
-              volumeMounts:
-                - name: results
-                  mountPath: /tmp/results
-              resources:
-                requests:
-                  memory: "2Gi"
-                  cpu: "200m"
-                limits:
-                  memory: "4Gi"
-                  cpu: "1000m"
-          volumes:
-            - name: results
-              emptyDir: {}
+# Two CronJobs run sequentially (staggered) so both endpoints are benchmarked without
+# doubling load. api.hyperbolic.xyz at :00,:20,:40; inference.hyperbolic.ai at :10,:30,:50.
+#
+# Workload is identical for both (same model, concurrency, duration, output tokens, timeout,
+# streaming). Only ENDPOINT_URL and auth differ (api = Bearer key, inference = Cloudflare Access).
+# This keeps endpoint performance comparison in Datadog apples-to-apples.
+---
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: aiperf-benchmark-api
+  namespace: inference-benchmark
+  labels:
+    app: aiperf
+    component: benchmark
+    endpoint: api.hyperbolic.xyz
+spec:
+  suspend: true  # set to false to resume benchmarks
+  schedule: "0,20,40 * * * *"
+  concurrencyPolicy: Forbid
+  successfulJobsHistoryLimit: 5
+  failedJobsHistoryLimit: 3
+  jobTemplate:
+    spec:
+      template:
+        metadata:
+          labels:
+            app: aiperf
+            component: benchmark
+        spec:
+          restartPolicy: OnFailure
+          # The benchmark never talks to the Kubernetes API, but a token was
+          # mounted into it by default — readable by anything that could run code
+          # in the container, and usable against the cluster.
+          automountServiceAccountToken: false
+          # Nothing here needs root. `fsGroup` matches the GID created in
+          # aiperf/Dockerfile so the results volume remains writable.
+          securityContext:
+            runAsNonRoot: true
+            runAsUser: 10001
+            runAsGroup: 10001
+            fsGroup: 10001
+            seccompProfile:
+              type: RuntimeDefault
+          imagePullSecrets:
+            - name: ghcr-image-pull-secret
+          containers:
+            - name: aiperf
+              image: ghcr.io/hyperboliclabs/aiperf:latest
+              imagePullPolicy: Always
+              # `readOnlyRootFilesystem` is deliberately not set: aiperf writes
+              # scratch files outside the mounted results directory, so enabling
+              # it needs an additional emptyDir over /tmp and a verified run.
+              securityContext:
+                allowPrivilegeEscalation: false
+                capabilities:
+                  drop: ["ALL"]
+              command:
+                - /bin/sh
+                - -c
+                - |
+                  set -eu
+                  echo "=== Running AIPerf Benchmark (api.hyperbolic.xyz) ==="
+                  python3 /scripts/benchmark.py
+                  echo "=== Benchmark Complete ==="
+                  ls -lh /tmp/results/ || echo "No results directory"
+              env:
+                - name: MODEL_NAME
+                  value: "Qwen/Qwen3-VL-32B-Thinking"
+                - name: ENDPOINT_URL
+                  value: "https://api.hyperbolic.xyz"
+                - name: ENDPOINT_TYPE
+                  value: "chat"
+                - name: OPENAI_API_KEY
+                  valueFrom:
+                    secretKeyRef:
+                      name: hyperbolic-api-key
+                      key: api-key
+                      optional: true
+                - name: CF_ACCESS_CLIENT_ID
+                  valueFrom:
+                    secretKeyRef:
+                      name: cloudflare-access-credentials
+                      key: client-id
+                      optional: true
+                - name: CF_ACCESS_CLIENT_SECRET
+                  valueFrom:
+                    secretKeyRef:
+                      name: cloudflare-access-credentials
+                      key: client-secret
+                      optional: true
+                - name: CONCURRENCY
+                  value: "10"
+                - name: BENCHMARK_DURATION
+                  value: "480"
+                - name: BENCHMARK_GRACE_PERIOD
+                  value: "30"
+                - name: REQUEST_TIMEOUT
+                  value: "60"
+                - name: OUTPUT_TOKENS_MEAN
+                  value: "50"
+                - name: REQUEST_COUNT
+                  value: "10000"
+                - name: STREAMING
+                  value: "true"
+                - name: OUTPUT_DIR
+                  value: "/tmp/results"
+                - name: TERM
+                  value: "dumb"
+                - name: CI
+                  value: "true"
+                - name: NO_COLOR
+                  value: "1"
+                - name: PYTHONUNBUFFERED
+                  value: "1"
+                - name: AIPERF_SERVICE_PROFILE_CONFIGURE_TIMEOUT
+                  value: "600.0"
+                - name: AIPERF_SERVICE_PROFILE_START_TIMEOUT
+                  value: "300.0"
+                - name: AIPERF_DATASET_CONFIGURATION_TIMEOUT
+                  value: "600.0"
+                - name: DD_API_KEY
+                  valueFrom:
+                    secretKeyRef:
+                      name: datadog-api-key
+                      key: api-key
+                      optional: true
+                - name: DD_SITE
+                  value: "us5.datadoghq.com"
+              volumeMounts:
+                - name: results
+                  mountPath: /tmp/results
+              resources:
+                requests:
+                  memory: "2Gi"
+                  cpu: "200m"
+                limits:
+                  memory: "4Gi"
+                  cpu: "1000m"
+          volumes:
+            - name: results
+              emptyDir: {}
+---
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: aiperf-benchmark-inference
+  namespace: inference-benchmark
+  labels:
+    app: aiperf
+    component: benchmark
+    endpoint: inference.hyperbolic.ai
+spec:
+  suspend: true  # set to false to resume benchmarks
+  schedule: "10,30,50 * * * *"
+  concurrencyPolicy: Forbid
+  successfulJobsHistoryLimit: 5
+  failedJobsHistoryLimit: 3
+  jobTemplate:
+    spec:
+      template:
+        metadata:
+          labels:
+            app: aiperf
+            component: benchmark
+        spec:
+          restartPolicy: OnFailure
+          # See the api CronJob above: no Kubernetes API access is needed, and the
+          # workload runs as an unprivileged user with a matching fsGroup.
+          automountServiceAccountToken: false
+          securityContext:
+            runAsNonRoot: true
+            runAsUser: 10001
+            runAsGroup: 10001
+            fsGroup: 10001
+            seccompProfile:
+              type: RuntimeDefault
+          imagePullSecrets:
+            - name: ghcr-image-pull-secret
+          containers:
+            - name: aiperf
+              image: ghcr.io/hyperboliclabs/aiperf:latest
+              imagePullPolicy: Always
+              securityContext:
+                allowPrivilegeEscalation: false
+                capabilities:
+                  drop: ["ALL"]
+              command:
+                - /bin/sh
+                - -c
+                - |
+                  set -eu
+                  echo "=== Running AIPerf Benchmark (inference.hyperbolic.ai) ==="
+                  python3 /scripts/benchmark.py
+                  echo "=== Benchmark Complete ==="
+                  ls -lh /tmp/results/ || echo "No results directory"
+              env:
+                - name: MODEL_NAME
+                  value: "Qwen/Qwen3-VL-32B-Thinking"
+                - name: ENDPOINT_URL
+                  value: "https://inference.hyperbolic.ai"
+                - name: ENDPOINT_TYPE
+                  value: "chat"
+                # inference.hyperbolic.ai uses Cloudflare Access (no OPENAI_API_KEY)
+                - name: CF_ACCESS_CLIENT_ID
+                  valueFrom:
+                    secretKeyRef:
+                      name: cloudflare-access-credentials
+                      key: client-id
+                - name: CF_ACCESS_CLIENT_SECRET
+                  valueFrom:
+                    secretKeyRef:
+                      name: cloudflare-access-credentials
+                      key: client-secret
+                - name: CONCURRENCY
+                  value: "10"
+                - name: BENCHMARK_DURATION
+                  value: "480"
+                - name: BENCHMARK_GRACE_PERIOD
+                  value: "30"
+                - name: REQUEST_TIMEOUT
+                  value: "60"
+                - name: OUTPUT_TOKENS_MEAN
+                  value: "50"
+                - name: REQUEST_COUNT
+                  value: "10000"
+                - name: STREAMING
+                  value: "true"
+                - name: OUTPUT_DIR
+                  value: "/tmp/results"
+                - name: TERM
+                  value: "dumb"
+                - name: CI
+                  value: "true"
+                - name: NO_COLOR
+                  value: "1"
+                - name: PYTHONUNBUFFERED
+                  value: "1"
+                - name: AIPERF_SERVICE_PROFILE_CONFIGURE_TIMEOUT
+                  value: "600.0"
+                - name: AIPERF_SERVICE_PROFILE_START_TIMEOUT
+                  value: "300.0"
+                - name: AIPERF_DATASET_CONFIGURATION_TIMEOUT
+                  value: "600.0"
+                - name: DD_API_KEY
+                  valueFrom:
+                    secretKeyRef:
+                      name: datadog-api-key
+                      key: api-key
+                      optional: true
+                - name: DD_SITE
+                  value: "us5.datadoghq.com"
+              volumeMounts:
+                - name: results
+                  mountPath: /tmp/results
+              resources:
+                requests:
+                  memory: "2Gi"
+                  cpu: "200m"
+                limits:
+                  memory: "4Gi"
+                  cpu: "1000m"
+          volumes:
+            - name: results
+              emptyDir: {}