diff --git a/CHANGELOG.md b/CHANGELOG.md index 2c0b495..47a49e7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,61 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [1.1.0] - 2026-07-30 + +Ports integration fixes and compatibility improvements developed in the +`whitebox-xai-azure` monorepo's `sdk/` directory since the 1.0.0 cut. + +### Fixed +- **XGBoost/LightGBM**: `XGBoostMonitor.predict()`/`LightGBMMonitor.predict()` + and `wrap_xgboost_model()`/`wrap_lightgbm_model()` called a nonexistent + `log_predictions()` method (raising `AttributeError`) instead of + `ModelMonitor.log_batch()` — the primary documented usage of both + integrations was broken. +- **TensorFlow/Keras**: `KerasMonitor.predict()` called `log_prediction()`/ + `log_batch()` with invalid keyword arguments (`prediction=`, `actual=`, + `predictions=`, `actuals=`), raising `TypeError` on every call. + `KerasMonitor.log_epoch()`/`log_checkpoint()` (and the + `WhiteBoxXAICallback` training callback) called a `log_custom_metric()` + method that didn't exist, raising `AttributeError`. + `KerasMonitor.set_baseline()` raised `TypeError` when computing baseline + predictions, since it called the parent `ModelMonitor.set_baseline()` + with two positional arguments against a signature that only accepted one + (see the `ModelMonitor.set_baseline()` fix below). + `register_saved_model()` raised `ValueError` unless `self.model` was set, + even though it exists specifically to register a model saved to disk. + `wrap_keras_model()` also called `log_batch()` with invalid keyword + arguments. +- **Hugging Face Transformers**: `TransformersMonitor.log_prediction_transformers()`/ + `log_batch_transformers()` called `log_prediction()`/`log_batch()` with + invalid keyword arguments, raising `TypeError` on every prediction. + `TransformersMonitor.set_baseline()` had the same `ModelMonitor.set_baseline()` + argument-count crash as `KerasMonitor` above. + `wrap_transformers_pipeline()` reassigned `pipeline.__call__` on a pipeline + *instance*, which Python never actually invokes (dunder-method lookup + happens on the type, not the instance) — wrapped pipelines silently never + logged any predictions. +- **LangChain**: `LangChainMonitor.log_chain_execution()`, + `log_agent_execution()`, `log_llm_call()`, `log_tool_call()`, and + `log_rag_retrieval()` all passed `prediction=` to `log_prediction()`, + which only accepts `output=` — all five convenience-logging methods + raised `TypeError`. +- `OfflineQueue.clear_completed()` used an exclusive day-boundary comparison + (`created_at < ...`), so a completed record exactly `older_than_days` old + was never cleared. Now inclusive (`<=`). + +### Added +- `whiteboxxai.integrations.langchain`/`langchain_agents` now try + `langchain_core.*` first (the modern, lighter-weight package) and fall + back to the legacy `langchain.callbacks.base`/`langchain.schema.*` + locations, so the SDK works with `langchain-core`-only installs. + `AgentExecutor`/`Chain` (higher-level `langchain`-only abstractions) are + probed independently so their absence no longer disables the rest of the + integration. +- `ModelMonitor.set_baseline()` accepts an optional `labels` argument for + baseline predictions/labels. +- `KerasMonitor.log_custom_metric()`. + ## [1.0.0] - 2026-07-14 First stable, production release. This release realigns the SDK with the @@ -153,6 +208,9 @@ from the git history between the `0.2.0` and `0.2.1` tags: - PII detection and masking - Secure API key handling -[Unreleased]: https://github.com/AgentaFlow/whitebox-python-sdk/compare/v0.2.0...HEAD -[0.2.0]: https://github.com/AgentaFlow/whitebox-python-sdk/compare/v0.1.0...v0.2.0 -[0.1.0]: https://github.com/AgentaFlow/whitebox-python-sdk/releases/tag/v0.1.0 +[Unreleased]: https://github.com/AgentaFlow/whitebox-python-sdk/compare/1.1.0...HEAD +[1.1.0]: https://github.com/AgentaFlow/whitebox-python-sdk/compare/1.0.0...1.1.0 +[1.0.0]: https://github.com/AgentaFlow/whitebox-python-sdk/compare/0.2.1...1.0.0 +[0.2.1]: https://github.com/AgentaFlow/whitebox-python-sdk/compare/0.2.0...0.2.1 +[0.2.0]: https://github.com/AgentaFlow/whitebox-python-sdk/compare/0.1.0...0.2.0 +[0.1.0]: https://github.com/AgentaFlow/whitebox-python-sdk/releases/tag/0.1.0 diff --git a/docs/api-reference.md b/docs/api-reference.md index 7284b6c..d01b56e 100644 --- a/docs/api-reference.md +++ b/docs/api-reference.md @@ -141,7 +141,7 @@ log_batch(predictions: List[dict]) Log multiple predictions. ```python -set_baseline(data: np.ndarray, **kwargs) +set_baseline(data: np.ndarray, labels: Optional[np.ndarray] = None) ``` Set baseline data for drift detection. diff --git a/pyproject.toml b/pyproject.toml index d475d85..7889512 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "whitebox-xai-sdk" -version = "1.0.0" +version = "1.1.0" description = "Python SDK for WhiteBox XAI - AI Observability & Explainability Platform" readme = "README.md" authors = [ diff --git a/tests/unit/test_monitor.py b/tests/unit/test_monitor.py index ed77de1..8fe8609 100644 --- a/tests/unit/test_monitor.py +++ b/tests/unit/test_monitor.py @@ -6,6 +6,7 @@ from unittest.mock import AsyncMock, Mock, patch +import numpy as np import pytest from whiteboxxai.client import WhiteBoxXAI @@ -178,6 +179,33 @@ def test_get_drift_reports(self): assert reports[0]["drift_score"] == 0.8 +class TestBaselineData: + """Tests for ModelMonitor.set_baseline().""" + + def test_set_baseline_data_only(self): + """Test setting baseline data without labels.""" + mock_client = Mock(spec=WhiteBoxXAI) + monitor = ModelMonitor(client=mock_client, model_id="model-123") + data = np.array([[1.0, 2.0], [3.0, 4.0]]) + + monitor.set_baseline(data) + + assert monitor._baseline_data is data + assert monitor._baseline_labels is None + + def test_set_baseline_with_labels(self): + """Test setting baseline data with labels.""" + mock_client = Mock(spec=WhiteBoxXAI) + monitor = ModelMonitor(client=mock_client, model_id="model-123") + data = np.array([[1.0, 2.0], [3.0, 4.0]]) + labels = np.array([0, 1]) + + monitor.set_baseline(data, labels=labels) + + assert monitor._baseline_data is data + assert monitor._baseline_labels is labels + + class TestAlertManagement: """Tests for alert management.""" diff --git a/tests/unit/test_offline.py b/tests/unit/test_offline.py new file mode 100644 index 0000000..feabc9a --- /dev/null +++ b/tests/unit/test_offline.py @@ -0,0 +1,45 @@ +""" +Tests for SDK Offline Queue Module + +Tests for offline mode queueing functionality. +""" + +import sqlite3 + +from whiteboxxai.offline import OfflineQueue, OperationPriority, OperationType + + +class TestOfflineQueueClearCompleted: + """Tests for OfflineQueue.clear_completed().""" + + def test_clears_records_exactly_older_than_days_old(self, tmp_path): + """A completed record whose age is exactly `older_than_days` must be + cleared (boundary is inclusive: created_at <= cutoff, not <).""" + db_path = str(tmp_path / "queue.db") + queue = OfflineQueue(db_path=db_path) + + op_id = queue.enqueue(OperationType.PREDICT, {"id": 1}, OperationPriority.NORMAL) + queue.mark_success(op_id) + + with sqlite3.connect(db_path) as conn: + conn.execute( + "UPDATE queue SET created_at = datetime('now', '-7 days') WHERE id = ?", + (op_id,), + ) + conn.commit() + + queue.clear_completed(older_than_days=7) + + assert queue.get_statistics()["total"] == 0 + + def test_keeps_records_younger_than_older_than_days(self, tmp_path): + """A completed record younger than `older_than_days` must be kept.""" + db_path = str(tmp_path / "queue.db") + queue = OfflineQueue(db_path=db_path) + + op_id = queue.enqueue(OperationType.PREDICT, {"id": 1}, OperationPriority.NORMAL) + queue.mark_success(op_id) + + queue.clear_completed(older_than_days=7) + + assert queue.get_statistics()["total"] == 1 diff --git a/whiteboxxai/integrations/boosting.py b/whiteboxxai/integrations/boosting.py index 1362160..cfd507b 100644 --- a/whiteboxxai/integrations/boosting.py +++ b/whiteboxxai/integrations/boosting.py @@ -64,6 +64,31 @@ lgb = None +def _build_prediction_batch( + inputs: np.ndarray, + predictions: np.ndarray, + actuals: Optional[np.ndarray] = None, + probabilities: Optional[np.ndarray] = None, +) -> List[Dict[str, Any]]: + """Build per-sample dicts for ModelMonitor.log_batch() from parallel arrays.""" + + def _item(value: Any) -> Any: + return value.tolist() if isinstance(value, np.ndarray) else value + + batch = [] + for i in range(len(inputs)): + item: Dict[str, Any] = { + "inputs": _item(inputs[i]), + "output": _item(predictions[i]), + } + if actuals is not None: + item["actual"] = _item(actuals[i]) + if probabilities is not None: + item["probability"] = _item(probabilities[i]) + batch.append(item) + return batch + + class XGBoostMonitor(ModelMonitor): """ Monitor for XGBoost models. @@ -241,23 +266,13 @@ def predict( # Log predictions if log_predictions: - pred_metadata = metadata or {} - - # Add feature importance for this batch if enabled - if self.track_feature_importance: - try: - importance = self._get_feature_importance(model) - if importance: - pred_metadata["feature_importance"] = importance - except Exception as e: - warnings.warn(f"Failed to extract feature importance: {e}") - - self.log_predictions( - inputs=X, - predictions=predictions, - actuals=y_true, - probabilities=probabilities, - metadata=pred_metadata, + self.log_batch( + _build_prediction_batch( + inputs=X, + predictions=predictions, + actuals=y_true, + probabilities=probabilities, + ) ) return predictions @@ -476,23 +491,13 @@ def predict( # Log predictions if log_predictions: - pred_metadata = metadata or {} - - # Add feature importance for this batch if enabled - if self.track_feature_importance: - try: - importance = self._get_feature_importance(model) - if importance: - pred_metadata["feature_importance"] = importance - except Exception as e: - warnings.warn(f"Failed to extract feature importance: {e}") - - self.log_predictions( - inputs=X, - predictions=predictions, - actuals=y_true, - probabilities=probabilities, - metadata=pred_metadata, + self.log_batch( + _build_prediction_batch( + inputs=X, + predictions=predictions, + actuals=y_true, + probabilities=probabilities, + ) ) return predictions @@ -558,7 +563,7 @@ def wrap_xgboost_model(model: Any, monitor: XGBoostMonitor, auto_register: bool >>> wrapped_model = wrap_xgboost_model(model, monitor) >>> predictions = wrapped_model.predict(X_test) # Auto-logged """ - if auto_register and not monitor._model_id: + if auto_register and not monitor.model_id: monitor.register_from_model(model) # Store original methods @@ -574,7 +579,7 @@ def wrapped_predict(X, *args, **kwargs): # Log predictions try: - monitor.log_predictions(inputs=X, predictions=predictions) + monitor.log_batch(_build_prediction_batch(inputs=X, predictions=predictions)) except Exception as e: warnings.warn(f"Failed to log predictions: {e}") @@ -589,8 +594,10 @@ def wrapped_predict_proba(X, *args, **kwargs): # Log with probabilities try: predictions = np.argmax(probabilities, axis=1) - monitor.log_predictions( - inputs=X, predictions=predictions, probabilities=probabilities + monitor.log_batch( + _build_prediction_batch( + inputs=X, predictions=predictions, probabilities=probabilities + ) ) except Exception as e: warnings.warn(f"Failed to log predictions: {e}") @@ -624,7 +631,7 @@ def wrap_lightgbm_model(model: Any, monitor: LightGBMMonitor, auto_register: boo >>> wrapped_model = wrap_lightgbm_model(model, monitor) >>> predictions = wrapped_model.predict(X_test) # Auto-logged """ - if auto_register and not monitor._model_id: + if auto_register and not monitor.model_id: monitor.register_from_model(model) # Store original methods @@ -640,7 +647,7 @@ def wrapped_predict(X, *args, **kwargs): # Log predictions try: - monitor.log_predictions(inputs=X, predictions=predictions) + monitor.log_batch(_build_prediction_batch(inputs=X, predictions=predictions)) except Exception as e: warnings.warn(f"Failed to log predictions: {e}") @@ -655,8 +662,10 @@ def wrapped_predict_proba(X, *args, **kwargs): # Log with probabilities try: predictions = np.argmax(probabilities, axis=1) - monitor.log_predictions( - inputs=X, predictions=predictions, probabilities=probabilities + monitor.log_batch( + _build_prediction_batch( + inputs=X, predictions=predictions, probabilities=probabilities + ) ) except Exception as e: warnings.warn(f"Failed to log predictions: {e}") diff --git a/whiteboxxai/integrations/langchain.py b/whiteboxxai/integrations/langchain.py index ae00904..669acb0 100644 --- a/whiteboxxai/integrations/langchain.py +++ b/whiteboxxai/integrations/langchain.py @@ -10,21 +10,40 @@ import warnings from typing import Any, Dict, List, Optional +# langchain_core is the modern, lighter-weight home for these types and may +# be installed without the full legacy langchain package. Try it first, then +# fall back to the legacy langchain.callbacks.base/langchain.schema locations. try: - from langchain.agents import AgentExecutor - from langchain.callbacks.base import BaseCallbackHandler - from langchain.chains.base import Chain - from langchain.schema import AgentAction, AgentFinish, LLMResult + from langchain_core.agents import AgentAction, AgentFinish + from langchain_core.callbacks import BaseCallbackHandler + from langchain_core.outputs import LLMResult LANGCHAIN_AVAILABLE = True except ImportError: - LANGCHAIN_AVAILABLE = False - BaseCallbackHandler = object - Chain = object + try: + from langchain.callbacks.base import BaseCallbackHandler + from langchain.schema import AgentAction, AgentFinish, LLMResult + + LANGCHAIN_AVAILABLE = True + except ImportError: + LANGCHAIN_AVAILABLE = False + BaseCallbackHandler = object + AgentAction = object + AgentFinish = object + LLMResult = object + +# AgentExecutor/Chain are higher-level `langchain` package abstractions, not +# part of langchain_core, so they may be unavailable even when langchain_core +# (and thus LANGCHAIN_AVAILABLE) is -- probe them separately. +try: + from langchain.agents import AgentExecutor +except ImportError: AgentExecutor = object - AgentAction = object - AgentFinish = object - LLMResult = object + +try: + from langchain.chains.base import Chain +except ImportError: + Chain = object from whiteboxxai.monitor import ModelMonitor @@ -198,7 +217,7 @@ def log_chain_execution( # Log as prediction self.log_prediction( inputs=inputs, - prediction=outputs, + output=outputs, metadata=metadata, ) @@ -233,7 +252,7 @@ def log_agent_execution( # Log as prediction self.log_prediction( inputs=inputs, - prediction=outputs, + output=outputs, metadata=metadata, ) @@ -269,7 +288,7 @@ def log_llm_call( self.log_prediction( inputs={"prompt": prompt}, - prediction={"response": response}, + output={"response": response}, metadata=metadata, ) @@ -299,7 +318,7 @@ def log_tool_call( self.log_prediction( inputs={"tool_input": tool_input}, - prediction={"tool_output": tool_output}, + output={"tool_output": tool_output}, metadata=metadata, ) @@ -332,7 +351,7 @@ def log_rag_retrieval( self.log_prediction( inputs={"query": query}, - prediction={"documents": documents}, + output={"documents": documents}, metadata=metadata, ) diff --git a/whiteboxxai/integrations/langchain_agents.py b/whiteboxxai/integrations/langchain_agents.py index a757701..b7d375b 100644 --- a/whiteboxxai/integrations/langchain_agents.py +++ b/whiteboxxai/integrations/langchain_agents.py @@ -11,8 +11,16 @@ from datetime import datetime from typing import Any, Dict, List, Optional, Union -from langchain.callbacks.base import BaseCallbackHandler -from langchain.schema import AgentAction, AgentFinish, LLMResult +# langchain_core is the modern, lighter-weight home for these types; fall +# back to the legacy langchain.schema.* locations for older installs (mirrors +# the version-tolerant pattern in integrations/langchain.py). +try: + from langchain_core.agents import AgentAction, AgentFinish + from langchain_core.callbacks import BaseCallbackHandler + from langchain_core.outputs import LLMResult +except ImportError: + from langchain.callbacks.base import BaseCallbackHandler + from langchain.schema import AgentAction, AgentFinish, LLMResult try: from whiteboxxai import WhiteBoxXAI diff --git a/whiteboxxai/integrations/tensorflow.py b/whiteboxxai/integrations/tensorflow.py index 127d40a..b32ac01 100644 --- a/whiteboxxai/integrations/tensorflow.py +++ b/whiteboxxai/integrations/tensorflow.py @@ -211,22 +211,26 @@ def predict( # Determine if batch or single if len(inputs_np.shape) == 1 or inputs_np.shape[0] == 1: # Single prediction + metadata = dict(kwargs) if kwargs else {} + if actuals is not None: + metadata["actual"] = actuals[0] self.log_prediction( inputs=inputs_np[0] if len(inputs_np.shape) > 1 else inputs_np, - prediction=predictions_np[0] - if len(predictions_np.shape) > 1 - else predictions_np, - actual=actuals[0] if actuals is not None else None, - **kwargs, + output=(predictions_np[0] if len(predictions_np.shape) > 1 else predictions_np), + metadata=metadata or None, ) else: # Batch prediction - self.log_batch( - inputs=inputs_np, - predictions=predictions_np, - actuals=actuals, - **kwargs, - ) + batch = [] + for i in range(len(inputs_np)): + item: Dict[str, Any] = { + "inputs": inputs_np[i], + "output": predictions_np[i], + } + if actuals is not None: + item["actual"] = actuals[i] + batch.append(item) + self.log_batch(batch) return predictions_np @@ -259,6 +263,19 @@ def set_baseline( # Set baseline through parent class super().set_baseline(baseline_data, baseline_predictions) + def log_custom_metric(self, name: str, data: Dict[str, Any]) -> None: + """ + Log a custom named metric (e.g. training epoch, checkpoint). + + There is no dedicated backend endpoint for arbitrary custom metrics + yet, so this logs locally rather than silently dropping the call. + + Args: + name: Metric name/category + data: Metric payload + """ + logger.info(f"[{name}] {data}") + def log_epoch( self, epoch: int, @@ -319,13 +336,20 @@ def register_saved_model( metadata: Additional metadata """ model_metadata = { + "framework": "tensorflow", "model_path": model_path, "format": "SavedModel", **(metadata or {}), } if not self._registered: - self.register_from_model(**model_metadata) + self.model_id = self.client.register_model( + name=self._model_name or "saved_model", + model_type=self._model_type, + framework="tensorflow", + metadata=model_metadata, + ) + self._registered = True class WhiteBoxXAICallback(keras.callbacks.Callback): @@ -467,10 +491,7 @@ def logged_predict(x, *args, **kwargs): else: pred_np = predictions - monitor.log_batch( - inputs=x_np, - predictions=pred_np, - ) + monitor.log_batch([{"inputs": x_np[i], "output": pred_np[i]} for i in range(len(x_np))]) except Exception as e: warnings.warn(f"Failed to log predictions: {e}") diff --git a/whiteboxxai/integrations/transformers.py b/whiteboxxai/integrations/transformers.py index 4e4db44..db823b0 100644 --- a/whiteboxxai/integrations/transformers.py +++ b/whiteboxxai/integrations/transformers.py @@ -275,11 +275,11 @@ def log_prediction_transformers( # Log to WhiteBoxXAI self.log_prediction( inputs={"text": input_text}, - prediction=pred_value, - actual=actual, + output=pred_value, metadata={ "task": self._task, "raw_prediction": str(prediction), + "actual": actual, **kwargs, }, ) @@ -301,22 +301,23 @@ def log_batch_transformers( **kwargs: Additional metadata """ # Prepare batch data - batch_inputs = [{"text": text} for text in inputs] batch_predictions = [ self._extract_prediction_value(pred) if isinstance(pred, dict) else pred for pred in predictions ] + batch = [] + for i, text in enumerate(inputs): + item: Dict[str, Any] = { + "inputs": {"text": text}, + "output": batch_predictions[i], + } + if actuals is not None: + item["actual"] = actuals[i] + batch.append(item) + # Log batch - self.log_batch( - inputs=batch_inputs, - predictions=batch_predictions, - actuals=actuals, - metadata={ - "task": self._task, - **kwargs, - }, - ) + self.log_batch(batch) def _extract_prediction_value(self, prediction: Dict) -> Any: """Extract the main prediction value from a pipeline output.""" @@ -494,34 +495,43 @@ def wrap_transformers_pipeline( result = wrapped("Great product!") ``` """ - original_call = pipeline.__call__ + # Python looks up special methods like __call__ on the type, not the + # instance, so `pipeline.__call__ = logged_call` would never actually + # change what `pipeline(...)` does. Wrap in a small object whose own + # __call__ is a real class method instead. + return _LoggedPipelineCall(pipeline, monitor) - def logged_call(*args, **kwargs): - # Make prediction - result = original_call(*args, **kwargs) - # Log to WhiteBoxXAI +class _LoggedPipelineCall: + """Callable wrapper that logs each pipeline call before returning its result.""" + + def __init__(self, pipeline: Pipeline, monitor: "TransformersMonitor"): + self._pipeline = pipeline + self._monitor = monitor + + def __call__(self, *args, **kwargs): + result = self._pipeline(*args, **kwargs) + try: inputs = args[0] if args else kwargs.get("inputs") if isinstance(inputs, list): - monitor.log_batch_transformers( + self._monitor.log_batch_transformers( inputs=inputs, predictions=result, ) else: - monitor.log_prediction_transformers( + self._monitor.log_prediction_transformers( input_text=inputs, prediction=result, ) except Exception as e: - warnings.warn(f"Failed to log predictions: {e}") + warnings.warn(f"Failed to log prediction: {e}") return result - # Replace __call__ method - pipeline.__call__ = logged_call - return pipeline + def __getattr__(self, name): + return getattr(self._pipeline, name) __all__ = [ diff --git a/whiteboxxai/monitor.py b/whiteboxxai/monitor.py index ad0b2b5..692b945 100644 --- a/whiteboxxai/monitor.py +++ b/whiteboxxai/monitor.py @@ -70,6 +70,7 @@ def __init__( self._buffer: List[Dict[str, Any]] = [] self._prediction_count = 0 self._baseline_data: Optional[np.ndarray] = None + self._baseline_labels: Optional[np.ndarray] = None def __enter__(self) -> "ModelMonitor": return self @@ -344,14 +345,16 @@ def get_active_alerts(self) -> List[Dict[str, Any]]: """Get alert rules for this model.""" return self.client.alerts.list(model_id=self.model_id) - def set_baseline(self, data: np.ndarray) -> None: + def set_baseline(self, data: np.ndarray, labels: Optional[np.ndarray] = None) -> None: """ Set baseline data for drift detection. Args: data: Baseline data array + labels: Baseline labels/predictions, if available (optional) """ self._baseline_data = data + self._baseline_labels = labels def detect_drift( self, diff --git a/whiteboxxai/offline.py b/whiteboxxai/offline.py index e2b11f6..3674124 100644 --- a/whiteboxxai/offline.py +++ b/whiteboxxai/offline.py @@ -322,7 +322,7 @@ def clear_completed(self, older_than_days: int = 7): """ DELETE FROM queue WHERE status = 'completed' - AND created_at < datetime('now', '-' || ? || ' days') + AND created_at <= datetime('now', '-' || ? || ' days') """, (older_than_days,), )