diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index 5b06f245..822726fc 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -34,6 +34,8 @@ jobs:
run: make lint
- name: Check formatting
run: make format
+ - name: Check type annotations
+ run: make typecheck
test:
name: Build & Test
@@ -64,3 +66,31 @@ jobs:
python -m build
- name: Test and coverage
run: make coverage
+
+ pandas_compat:
+ name: Pandas Compat (${{ matrix.pandas }})
+ needs: code_quality
+ runs-on: ubuntu-latest
+ timeout-minutes: 60
+ strategy:
+ fail-fast: false
+ matrix:
+ pandas: ["2.2.2", "2.3.*", "3.*"]
+ steps:
+ - uses: actions/checkout@v6
+ with:
+ fetch-depth: 0 # required for setuptools-scm to read git tags
+ - name: Set up Python 3.12
+ uses: actions/setup-python@v6
+ with:
+ python-version: "3.12"
+ cache: pip
+ cache-dependency-path: pyproject.toml
+ - name: Install dependencies
+ run: |
+ python -m pip install --upgrade pip
+ python -m pip install '.[all]'
+ python -m pip install "pandas==${{ matrix.pandas }}"
+ python -c "import pandas; print(pandas.__version__)"
+ - name: Test and coverage
+ run: make coverage
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 3e71b4fe..316f8222 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -36,6 +36,18 @@ repos:
args: [--fix]
# Run the formatter
- id: ruff-format
+- repo: local
+ hooks:
+ - id: mypy
+ name: mypy
+ # Runs from the local dev venv (not an isolated pre-commit env) so it sees
+ # shapash's actual dependencies (dash, pandas, sklearn, ...); pyproject.toml
+ # sets ignore_missing_imports=true, which would silently mask them as Any
+ # otherwise. Requires `.venv` to be set up.
+ entry: mypy shapash
+ language: system
+ pass_filenames: false
+ files: ^shapash/.*\.py$
- repo: https://github.com/adamchainz/blacken-docs
rev: 1.20.0
hooks:
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 51bb5afa..4d0e15cf 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -113,6 +113,11 @@ ruff check
ruff format
```
+Check your type annotations with mypy:
+```
+mypy shapash
+```
+
## Commit your changes
We recommend committing with clear messages and grouping your commits by modifications dependencies.
diff --git a/Makefile b/Makefile
index 2c64ef25..77536c42 100644
--- a/Makefile
+++ b/Makefile
@@ -56,6 +56,9 @@ lint: ## check style with ruff
format: ## check formatting with ruff
ruff format --check
+typecheck: ## check type annotations with mypy
+ mypy shapash
+
test: ## run tests quickly with the default Python
pytest
diff --git a/pyproject.toml b/pyproject.toml
index dc45a50e..fa7c7a6f 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -39,8 +39,9 @@ dependencies = [
"matplotlib>=3.8.4",
"nbformat>5.8.0",
"numba>=0.60.0",
- "numpy>=2.0.0,<2.6.0",
- "pandas>=2.2.2,<3.0.4",
+ "numpy>=2.0.0",
+ # 3.0.4 yanked upstream: segfaults in datetime ops (pandas #66083)
+ "pandas>=2.2.2,!=3.0.4,<4.0.0",
"plotly>=5.0.0,<6.0.0",
"scikit-learn>=1.8.0,<1.9.0",
"scipy>=1.13.0",
diff --git a/shapash/backend/base_backend.py b/shapash/backend/base_backend.py
index c261e482..82b1363c 100644
--- a/shapash/backend/base_backend.py
+++ b/shapash/backend/base_backend.py
@@ -41,7 +41,7 @@ def __init__(self, model: Any, preprocessing: Any | None = None):
self.model = model
self.preprocessing = preprocessing
self.explain_data: Any = None
- self.state = None
+ self.state: Any = None
self._case, self._classes = check_model(model)
if self._case not in self.supported_cases:
raise ValueError(f"Model not supported by the backend as it does not cover {self._case} case")
diff --git a/shapash/backend/lime_backend.py b/shapash/backend/lime_backend.py
index c84723c4..fab6a6f9 100644
--- a/shapash/backend/lime_backend.py
+++ b/shapash/backend/lime_backend.py
@@ -1,3 +1,5 @@
+from collections.abc import Callable
+
try:
from lime import lime_tabular
@@ -115,7 +117,7 @@ def _explain_multiclass(
self,
x: pd.DataFrame,
feature_names: list,
- predict_fn: callable,
+ predict_fn: Callable,
num_classes: int,
) -> list[pd.DataFrame]:
"""
@@ -152,7 +154,7 @@ def _explain_binary_or_regression(
self,
x: pd.DataFrame,
feature_names: list,
- predict_fn: callable,
+ predict_fn: Callable,
) -> pd.DataFrame:
"""
Compute LIME contributions for binary classification or regression.
diff --git a/shapash/explainer/consistency.py b/shapash/explainer/consistency.py
index 2d33a232..04f63378 100644
--- a/shapash/explainer/consistency.py
+++ b/shapash/explainer/consistency.py
@@ -10,6 +10,7 @@
from sklearn.manifold import MDS
from shapash.style.style_utils import colors_loading, define_style, select_palette
+from shapash.utils.dtypes import text_like_columns
from shapash.utils.utils import adjust_title_height
@@ -576,8 +577,8 @@ def plot_pairwise_consistency(
if isinstance(self.preprocessing, OrdinalEncoder):
encoder = self.preprocessing
else:
- categorical_features = [col for col in x.columns if x[col].dtype == "object"]
- encoder = OrdinalEncoder(cols=categorical_features, handle_unknown="ignore", return_df=True).fit(x)
+ categorical_features = text_like_columns(x, strict_object=False)
+ encoder = OrdinalEncoder(cols=categorical_features, handle_unknown="return_nan", return_df=True).fit(x)
x = encoder.transform(x)
xaxis_title = (
diff --git a/shapash/explainer/smart_explainer.py b/shapash/explainer/smart_explainer.py
index b8a946ca..808b31b4 100644
--- a/shapash/explainer/smart_explainer.py
+++ b/shapash/explainer/smart_explainer.py
@@ -6,6 +6,7 @@
import logging
import shutil
import tempfile
+from typing import Any
import numpy as np
import pandas as pd
@@ -195,7 +196,7 @@ def __init__(
features_groups=None,
features_dict=None,
label_dict=None,
- title_story: str = None,
+ title_story: str | None = None,
palette_name=None,
colors_dict=None,
**backend_kwargs,
@@ -246,7 +247,7 @@ def __init__(
self.features_compacity = None
self.contributions = None
self.explain_data = None
- self.features_imp = None
+ self.features_imp: Any = None
def compile(
self,
@@ -386,7 +387,7 @@ def _compile_features_groups(self, features_groups):
raise AssertionError(f"Selected backend ({self.backend.name}) does not support groups of features.")
# Compute contributions for groups of features
self.contributions_groups = self.state.compute_grouped_contributions(self.contributions, features_groups)
- self.features_imp_groups = None
+ self.features_imp_groups: Any = None
# Update features dict with groups names
self._update_features_dict_with_groups(features_groups=features_groups)
# Compute t-sne projections for groups of features
@@ -478,7 +479,7 @@ def add(
y_target=None,
label_dict=None,
features_dict=None,
- title_story: str = None,
+ title_story: str | None = None,
columns_order=None,
additional_data=None,
additional_features_dict=None,
@@ -1400,7 +1401,7 @@ def compute_features_compacity(self, selection, distance, nb_features):
self.features_compacity = {"features_needed": features_needed, "distance_reached": distance_reached}
- def init_app(self, settings: dict = None):
+ def init_app(self, settings: dict | None = None):
"""
Initialize a SmartApp instance for the current SmartExplainer object.
@@ -1436,10 +1437,10 @@ def init_app(self, settings: dict = None):
def run_app(
self,
- port: int = None,
- host: str = None,
- title_story: str = None,
- settings: dict = None,
+ port: int | None = None,
+ host: str | None = None,
+ title_story: str | None = None,
+ settings: dict | None = None,
) -> CustomThread:
"""
Launch the Shapash interpretability WebApp associated with this SmartExplainer.
@@ -1502,13 +1503,7 @@ def run_app(
port = 8050
host_name = get_host_name()
wsgi_server = make_server(host, port, self.smartapp.server)
- server_instance = CustomThread(target=wsgi_server.serve_forever)
-
- def _kill():
- wsgi_server.shutdown()
- server_instance.killed = True
-
- server_instance.kill = _kill
+ server_instance = CustomThread(target=wsgi_server.serve_forever, on_kill=wsgi_server.shutdown)
if host_name is None:
host_name = host
elif host != DEFAULT_HOST:
diff --git a/shapash/explainer/smart_plotter.py b/shapash/explainer/smart_plotter.py
index 6a8e22a8..b1049227 100644
--- a/shapash/explainer/smart_plotter.py
+++ b/shapash/explainer/smart_plotter.py
@@ -38,6 +38,7 @@
adjust_title_height,
compute_digit_number,
compute_sorted_variables_interactions_list_indices,
+ format_missing_value,
maximum_difference_sort_value,
top_contributors,
truncate_str,
@@ -2345,7 +2346,7 @@ def clustering_by_explainability_plot(
for idx, row in df_pred.iterrows():
text = f"Id: {idx}
"
if el not in ["predictions", "targets", "errors"]:
- text += f"{el}: {row[el]}
"
+ text += f"{el}: {format_missing_value(row[el])}
"
text += f"Predicted Value: {row['proba_values']:.{self._round_digit}f}
"
if "error" in df_pred.columns:
text += f"Error: {row['error']:.{self._round_digit}f}
"
@@ -2360,19 +2361,30 @@ def clustering_by_explainability_plot(
if el not in ["predictions", "targets", "errors"]:
is_num = is_numeric_dtype(df_pred[el]) and not is_bool_dtype(df_pred[el])
n_unique = df_pred[el].nunique(dropna=True)
+ cluster_values = df_pred.loc[df_pred["cluster"] == c, el]
if is_num and n_unique > 5:
- mean_el = df_pred.loc[df_pred["cluster"] == c, el].mean()
- std_el = df_pred.loc[df_pred["cluster"] == c, el].std()
- hv_text_cluster += f"
{el} mean: {mean_el:.{compute_digit_number(mean_el, 3)}f}"
- hv_text_cluster += f"
{el} std: {std_el:.{compute_digit_number(std_el, 3)}f}"
+ mean_el = cluster_values.mean()
+ std_el = cluster_values.std()
+ if pd.isna(mean_el) or pd.isna(std_el):
+ hv_text_cluster += f"
{el} mean: {format_missing_value(mean_el)}"
+ hv_text_cluster += f"
{el} std: {format_missing_value(std_el)}"
+ else:
+ hv_text_cluster += (
+ f"
{el} mean: {mean_el:.{compute_digit_number(mean_el, 3)}f}"
+ )
+ hv_text_cluster += f"
{el} std: {std_el:.{compute_digit_number(std_el, 3)}f}"
else:
- top_element = df_pred.loc[df_pred["cluster"] == c, el].mode()[0]
- top_element_percentage = (
- np.sum(df_pred.loc[df_pred["cluster"] == c, el] == top_element)
- / df_pred.loc[df_pred["cluster"] == c, el].size
- * 100
+ # dropna=False so that null values can be reported as the top modality
+ top_element = cluster_values.mode(dropna=False).iloc[0]
+ if pd.isna(top_element):
+ top_element_count = cluster_values.isna().sum()
+ else:
+ top_element_count = np.sum(cluster_values == top_element)
+ top_element_percentage = top_element_count / cluster_values.size * 100
+ hv_text_cluster += (
+ f"
{el} top: {format_missing_value(top_element)}"
+ f" ({top_element_percentage:.1f}%)"
)
- hv_text_cluster += f"
{el} top: {top_element} ({top_element_percentage:.1f}%)"
mean_predicted_value = df_pred.loc[df_pred["cluster"] == c, "proba_values"].mean()
hv_text_cluster += f"
Mean predicted value: {mean_predicted_value:.{compute_digit_number(mean_predicted_value, 3)}f}"
if "error" in df_pred.columns:
@@ -2490,7 +2502,7 @@ def clustering_by_explainability_plot(
for idx, row in df_pred.iterrows():
text = f"Id: {idx}
"
if el not in ["predictions", "targets", "errors"]:
- text += f"{el}: {row[el]}
"
+ text += f"{el}: {format_missing_value(row[el])}
"
text += f"Predicted Value: {row['predict_value']:.{self._round_digit}f}
"
if "error" in df_pred.columns:
text += f"Error: {row['error']:.{compute_digit_number(row['error'])}f}
"
@@ -2504,19 +2516,27 @@ def clustering_by_explainability_plot(
if el not in ["predictions", "targets", "errors"]:
is_num = is_numeric_dtype(df_pred[el]) and not is_bool_dtype(df_pred[el])
n_unique = df_pred[el].nunique(dropna=True)
+ cluster_values = df_pred.loc[df_pred["cluster"] == c, el]
if is_num and n_unique > 5:
- mean_el = df_pred.loc[df_pred["cluster"] == c, el].mean()
- std_el = df_pred.loc[df_pred["cluster"] == c, el].std()
- hv_text_cluster += f"
{el} mean: {mean_el:.{compute_digit_number(mean_el, 3)}f}"
- hv_text_cluster += f"
{el} std: {std_el:.{compute_digit_number(std_el, 3)}f}"
+ mean_el = cluster_values.mean()
+ std_el = cluster_values.std()
+ if pd.isna(mean_el) or pd.isna(std_el):
+ hv_text_cluster += f"
{el} mean: {format_missing_value(mean_el)}"
+ hv_text_cluster += f"
{el} std: {format_missing_value(std_el)}"
+ else:
+ hv_text_cluster += f"
{el} mean: {mean_el:.{compute_digit_number(mean_el, 3)}f}"
+ hv_text_cluster += f"
{el} std: {std_el:.{compute_digit_number(std_el, 3)}f}"
else:
- top_element = df_pred.loc[df_pred["cluster"] == c, el].mode()[0]
- top_element_percentage = (
- np.sum(df_pred.loc[df_pred["cluster"] == c, el] == top_element)
- / df_pred.loc[df_pred["cluster"] == c, el].size
- * 100
+ # dropna=False so that null values can be reported as the top modality
+ top_element = cluster_values.mode(dropna=False).iloc[0]
+ if pd.isna(top_element):
+ top_element_count = cluster_values.isna().sum()
+ else:
+ top_element_count = np.sum(cluster_values == top_element)
+ top_element_percentage = top_element_count / cluster_values.size * 100
+ hv_text_cluster += (
+ f"
{el} top: {format_missing_value(top_element)} ({top_element_percentage:.1f}%)"
)
- hv_text_cluster += f"
{el} top: {top_element} ({top_element_percentage:.1f}%)"
mean_predicted_value = df_pred.loc[df_pred["cluster"] == c, "predict_value"].mean()
hv_text_cluster += f"
Mean predicted value: {mean_predicted_value:.{compute_digit_number(mean_predicted_value, 3)}f}"
if "error" in df_pred.columns:
diff --git a/shapash/plots/plot_bar_chart.py b/shapash/plots/plot_bar_chart.py
index cbb66b6d..9cd95f87 100644
--- a/shapash/plots/plot_bar_chart.py
+++ b/shapash/plots/plot_bar_chart.py
@@ -1,7 +1,7 @@
from plotly import graph_objs as go
from plotly.offline import plot
-from shapash.utils.utils import add_line_break, adjust_title_height, truncate_str
+from shapash.utils.utils import add_line_break, adjust_title_height, format_missing_value, truncate_str
def plot_bar_chart(
@@ -103,6 +103,7 @@ def plot_bar_chart(
bars = []
for num, expl in enumerate(zip(var_dict, x_val, contrib, strict=False)):
feat_name, x_val_el, contrib_value = expl
+ x_val_el = format_missing_value(x_val_el)
is_grouped = False
if x_val_el == "":
ylabel = f"{feat_name}"
@@ -116,7 +117,7 @@ def plot_bar_chart(
feat_groups_values = x_init[features_groups[group_name]].loc[index_value[0]]
hoverlabel = "
".join(
[
- f"{add_line_break(features_dict.get(f_name, f_name), 40, maxlen=120)} :{add_line_break(f_value, 40, maxlen=160)}"
+ f"{add_line_break(features_dict.get(f_name, f_name), 40, maxlen=120)} :{add_line_break(format_missing_value(f_value), 40, maxlen=160)}"
for f_name, f_value in feat_groups_values.to_dict().items()
]
)
diff --git a/shapash/plots/plot_contribution.py b/shapash/plots/plot_contribution.py
index 3e25c998..5bae989a 100644
--- a/shapash/plots/plot_contribution.py
+++ b/shapash/plots/plot_contribution.py
@@ -7,7 +7,13 @@
from plotly.subplots import make_subplots
from sklearn.neighbors import KernelDensity
-from shapash.utils.utils import add_line_break, adjust_title_height, truncate_str
+from shapash.utils.utils import (
+ MISSING_VALUE_DISPLAY,
+ add_line_break,
+ adjust_title_height,
+ format_missing_value,
+ truncate_str,
+)
from shapash.webapp.utils.utils import round_to_k
NAN_PLACEHOLDER_K = 0.2
@@ -166,7 +172,8 @@ def plot_scatter(
fig.add_trace(density_plot)
nan_mask_arr = pd.isna(feature_values.iloc[:, 0]).to_numpy()
- has_nan_numeric = bool(nan_mask_arr.any()) and feature_values.iloc[:, 0].dtype.kind in "biufc"
+ has_nan = bool(nan_mask_arr.any())
+ has_nan_numeric = has_nan and feature_values.iloc[:, 0].dtype.kind in "biufc"
marker = None
if has_nan_numeric:
non_nan_arr = feature_values_array[~nan_mask_arr].astype(float)
@@ -178,6 +185,11 @@ def plot_scatter(
nan_x = 0.0
feature_values_array = np.where(nan_mask_arr, nan_x, feature_values_array)
marker = {"symbol": np.where(nan_mask_arr, "x", "circle").tolist()}
+ elif has_nan:
+ # non-numeric columns: display null values as an explicit "missing" modality
+ feature_values_array = feature_values_array.astype(object).copy()
+ feature_values_array[nan_mask_arr] = MISSING_VALUE_DISPLAY
+ marker = {"symbol": np.where(nan_mask_arr, "x", "circle").tolist()}
fig.add_scatter(
x=feature_values_array,
@@ -199,9 +211,9 @@ def plot_scatter(
# The values are used in the hovertext and the indexes are used for
# the interactions between the graphics.
customdata_values = feature_values_array
- if has_nan_numeric:
+ if has_nan:
customdata_values = feature_values_array.astype(object).copy()
- customdata_values[nan_mask_arr] = "missing"
+ customdata_values[nan_mask_arr] = MISSING_VALUE_DISPLAY
customdata = np.stack((customdata_values, feature_values.index.values), axis=-1)
fig.update_traces(customdata=customdata, hovertemplate=hovertemplate)
@@ -325,7 +337,7 @@ def plot_violin(
for i, c in enumerate(xs):
if pd.isna(c):
is_c = feature_values.iloc[:, 0].isna()
- c_label = "missing"
+ c_label = MISSING_VALUE_DISPLAY
else:
is_c = feature_values.iloc[:, 0] == c
c_label = c
@@ -440,7 +452,7 @@ def plot_violin(
)
# To change ticktext
- xs_labels = ["missing" if pd.isna(x) else x for x in xs]
+ xs_labels = [format_missing_value(x) for x in xs]
_update_xaxis_labels(fig, xs_labels, zoom)
_update_contributions_fig(
@@ -754,8 +766,12 @@ def _add_violin_and_scatter(
x = _create_jittered_points(x, percentage_series, side=side)
if colorpoints is not None:
colorpoints_selected = colorpoints.loc[feature_cond].values.flatten()
+ # display null values as "missing" in the hover text
+ point_values = np.array(
+ [format_missing_value(v) for v in feature_values.loc[feature_cond].values.flatten()], dtype=object
+ )
customdata = np.stack(
- (feature_values.loc[feature_cond].values.flatten(), contributions.loc[feature_cond].index.values),
+ (point_values, contributions.loc[feature_cond].index.values),
axis=-1,
)
marker = None
diff --git a/shapash/plots/plot_correlations.py b/shapash/plots/plot_correlations.py
index 4c050c0c..7cf4f18f 100644
--- a/shapash/plots/plot_correlations.py
+++ b/shapash/plots/plot_correlations.py
@@ -8,6 +8,7 @@
from shapash.manipulation.summarize import compute_corr
from shapash.style.style_utils import define_style, get_palette
+from shapash.utils.dtypes import text_like_columns
from shapash.utils.utils import adjust_title_height, compute_top_correlations_features, suffix_duplicates
@@ -159,11 +160,18 @@ def prepare_corr_matrix(df_subset):
features_to_hide = list(features_to_hide)
if optimized:
- categorical_columns = df.select_dtypes(include=["object", "category"]).columns
+ # Avoid mutating the caller-provided dataframe when bucketing categories.
+ df = df.copy()
+ categorical_columns = text_like_columns(df, strict_object=True)
+ if facet_col:
+ categorical_columns = [col for col in categorical_columns if col != facet_col]
for col in categorical_columns:
top_categories = df[col].value_counts().nlargest(200).index
- df[col] = df[col].where(df[col].isin(top_categories), other="Other")
+ keep_mask = df[col].isna() | df[col].isin(top_categories)
+ if isinstance(df[col].dtype, pd.CategoricalDtype) and "Other" not in df[col].cat.categories:
+ df[col] = df[col].cat.add_categories(["Other"])
+ df[col] = df[col].where(keep_mask, other="Other")
if len(df) > 10000:
df = df.sample(n=10000, random_state=1)
diff --git a/shapash/plots/plot_line_comparison.py b/shapash/plots/plot_line_comparison.py
index 676461d5..0bb0b0e8 100644
--- a/shapash/plots/plot_line_comparison.py
+++ b/shapash/plots/plot_line_comparison.py
@@ -3,7 +3,7 @@
from plotly import graph_objs as go
from plotly.offline import plot
-from shapash.utils.utils import add_line_break, adjust_title_height, truncate_str
+from shapash.utils.utils import add_line_break, adjust_title_height, format_missing_value, truncate_str
def plot_line_comparison(
@@ -108,7 +108,7 @@ def plot_line_comparison(
f"Id: {add_line_break(id_i, 40, 160)}"
+ f"
{add_line_break(feat, 40, 160)}
"
+ f"Contribution: {contrib[i]:.4f}
Value: "
- + str(add_line_break(pred_x_val, 40, 160))
+ + str(add_line_break(format_missing_value(pred_x_val), 40, 160))
)
lines.append(
diff --git a/shapash/plots/plot_univariate.py b/shapash/plots/plot_univariate.py
index a7fd3282..f4c9c0c4 100644
--- a/shapash/plots/plot_univariate.py
+++ b/shapash/plots/plot_univariate.py
@@ -411,7 +411,7 @@ def plot_categorical_distribution(
else:
color = style_dict.get(col, random_color())
- customdata = subset.apply(
+ customdata = df_cat.apply(
lambda row: (
f"{col}: {row[col]}
"
f"Percentage: {format(row.Percent, f'.{max(0, compute_digit_number(row.Percent, 3))}f')}%"
diff --git a/shapash/report/common.py b/shapash/report/common.py
index 33a1f6e2..7ea1181c 100644
--- a/shapash/report/common.py
+++ b/shapash/report/common.py
@@ -1,11 +1,14 @@
import builtins
import os
+from collections.abc import Callable
from enum import Enum
from importlib import import_module
from numbers import Number
import pandas as pd
-from pandas.api.types import is_bool_dtype, is_numeric_dtype, is_string_dtype
+from pandas.api.types import is_bool_dtype, is_numeric_dtype
+
+from shapash.utils.dtypes import is_text_like
class VarType(Enum):
@@ -39,9 +42,7 @@ def series_dtype(s: pd.Series, cat_num_threshold: int = 15) -> VarType:
"""
if is_bool_dtype(s):
return VarType.TYPE_CAT
- elif is_string_dtype(s):
- return VarType.TYPE_CAT
- elif s.dtype.name == "object":
+ elif is_text_like(s, strict_object=True):
return VarType.TYPE_CAT
elif is_numeric_dtype(s):
if numeric_is_continuous(s, threshold=cat_num_threshold):
@@ -73,7 +74,7 @@ def numeric_is_continuous(s: pd.Series, threshold: int = 15) -> bool:
return n_unique > threshold
-def compute_col_types(df_all: pd.DataFrame | None) -> dict | None:
+def compute_col_types(df_all: pd.DataFrame | None) -> dict:
"""
Computes the type of each column and stores the result in a dict.
@@ -181,7 +182,7 @@ def display_value(value: float, thousands_separator: str = ",", decimal_separato
return value_str.replace("/thousands/", thousands_separator).replace("/decimal/", decimal_separator)
-def replace_dict_values(obj: dict, replace_fn: callable, *args) -> dict:
+def replace_dict_values(obj: dict, replace_fn: Callable, *args) -> dict:
"""
Recursively iterates over all values of obj and changes its values using the replace_fn
diff --git a/shapash/report/generation.py b/shapash/report/generation.py
index 24d9930f..ddd6c47f 100644
--- a/shapash/report/generation.py
+++ b/shapash/report/generation.py
@@ -3,6 +3,7 @@
"""
import os
+from typing import TYPE_CHECKING
import pandas as pd
import papermill as pm
@@ -10,10 +11,13 @@
from shapash.utils.utils import get_project_root
+if TYPE_CHECKING:
+ from shapash.explainer.smart_explainer import SmartExplainer
+
def execute_report(
working_dir: str,
- explainer: object,
+ explainer: "SmartExplainer",
project_info_file: str,
x_train: pd.DataFrame | None = None,
y_train: pd.DataFrame | None = None,
diff --git a/shapash/report/project_report.py b/shapash/report/project_report.py
index 34bb5b15..a5864501 100644
--- a/shapash/report/project_report.py
+++ b/shapash/report/project_report.py
@@ -4,6 +4,7 @@
import sys
from datetime import date
from numbers import Number
+from typing import cast
import jinja2
import numpy as np
@@ -88,7 +89,9 @@ def __init__(
self.x_init = self.explainer.x_init
self.config = config if config is not None else dict()
self.col_names = list(self.explainer.columns_dict.values())
- self.df_train_test = self._create_train_test_df(test=self.x_init, train=self.x_train_pre)
+ # x_init is always set on a compiled explainer, so `test` is never None here and
+ # `_create_train_test_df` cannot return None.
+ self.df_train_test = cast(pd.DataFrame, self._create_train_test_df(test=self.x_init, train=self.x_train_pre))
if self.explainer.y_pred is not None:
self.y_pred = np.array(self.explainer.y_pred.T)[0]
else:
@@ -98,22 +101,22 @@ def __init__(
self.target_name = target_name_train or target_name_test
if "max_points" in self.config.keys():
- self.max_points = config["max_points"]
+ self.max_points = self.config["max_points"]
else:
self.max_points = 200
if "display_interaction_plot" in self.config.keys():
- self.display_interaction_plot = config["display_interaction_plot"]
+ self.display_interaction_plot = self.config["display_interaction_plot"]
else:
self.display_interaction_plot = False
if "nb_top_interactions" in self.config.keys():
- self.nb_top_interactions = config["nb_top_interactions"]
+ self.nb_top_interactions = self.config["nb_top_interactions"]
else:
self.nb_top_interactions = 5
if "title_story" in self.config.keys():
- self.title_story = config["title_story"]
+ self.title_story = self.config["title_story"]
elif self.explainer.title_story != "":
self.title_story = self.explainer.title_story
else:
@@ -136,7 +139,7 @@ def __init__(
@staticmethod
def _get_values_and_name(
y: pd.DataFrame | pd.Series | list | None, default_name: str
- ) -> tuple[list, str] | tuple[None, None]:
+ ) -> tuple[list | None, str | None]:
"""
Extracts vales and column name from a Pandas Series, DataFrame, or assign a default
name if y is a list of values.
diff --git a/shapash/utils/category_encoder_backend.py b/shapash/utils/category_encoder_backend.py
index ece19c57..a77aaec3 100644
--- a/shapash/utils/category_encoder_backend.py
+++ b/shapash/utils/category_encoder_backend.py
@@ -109,10 +109,11 @@ def inv_transform_target(x_in, enc_target):
# print("Warning in inverse TargetEncoder - col " + str(name_target) + ": Multiple label for the same value, "
# "each label will be separate using : / ")
+ data_type = tgt_enc.get("data_type") or "object"
transco = {
"col": name_target,
"mapping": pd.Series(data=aggregate.index, index=aggregate.values),
- "data_type": "object",
+ "data_type": data_type,
}
x_in = inv_transform_ordinal(x_in, [transco])
return x_in
diff --git a/shapash/utils/check.py b/shapash/utils/check.py
index af1463a6..a072a52d 100644
--- a/shapash/utils/check.py
+++ b/shapash/utils/check.py
@@ -10,6 +10,12 @@
from shapash.utils.transform import check_transformers, preprocessing_tolist
+def _is_string_dtype_metadata(dtype_value):
+ if not isinstance(dtype_value, str):
+ return False
+ return dtype_value in {"object", "str", "string"} or dtype_value.startswith("string[")
+
+
def check_preprocessing(preprocessing=None):
"""
Check that all transformation of the preprocessing are supported.
@@ -395,11 +401,16 @@ def check_postprocessing(x, postprocessing=None):
raise ValueError("Case modification unknown. Available ones are 'lower', 'upper'.")
if isinstance(x, dict):
- if x[key] != "object":
- raise ValueError(f"Expected string object to modify with upper/lower method in {key} dict")
+ if not _is_string_dtype_metadata(x[key]):
+ raise ValueError(
+ f"Expected string dtype metadata (object/str/string/string[...]) "
+ f"to apply upper/lower in {key} dict, got {x[key]!r}"
+ )
else:
if not pd.api.types.is_string_dtype(x[key]):
- raise ValueError(f"Expected string object to modify with upper/lower method in {key} dict")
+ raise ValueError(
+ f"Expected a string dtype to apply upper/lower on column {key}, got {x[key].dtype!r}"
+ )
if dict_post["type"] == "regex":
if set(dict_post["rule"].keys()) != {"in", "out"}:
@@ -408,11 +419,16 @@ def check_postprocessing(x, postprocessing=None):
f" must be 'in' and 'out'."
)
if isinstance(x, dict):
- if x[key] != "object":
- raise ValueError(f"Expected string object to modify with regex methods in {key} dict")
+ if not _is_string_dtype_metadata(x[key]):
+ raise ValueError(
+ f"Expected string dtype metadata (object/str/string/string[...]) "
+ f"to apply regex methods in {key} dict, got {x[key]!r}"
+ )
else:
if not pd.api.types.is_string_dtype(x[key]):
- raise ValueError(f"Expected string object to modify with upper/lower method in {key} dict")
+ raise ValueError(
+ f"Expected a string dtype to apply regex methods on column {key}, got {x[key].dtype!r}"
+ )
def check_features_name(columns_dict, features_dict, features):
diff --git a/shapash/utils/clustering.py b/shapash/utils/clustering.py
index b6246934..3d8b09ff 100644
--- a/shapash/utils/clustering.py
+++ b/shapash/utils/clustering.py
@@ -11,6 +11,7 @@
from sklearn.manifold import TSNE
from sklearn.preprocessing import LabelEncoder
+from shapash.utils.dtypes import is_text_like
from shapash.utils.utils import adjust_title_height
logger = logging.getLogger(__name__)
@@ -529,7 +530,7 @@ def encode_color_value(color_value):
label_mapping : dict or None
"""
- is_categorical = color_value.dtype == "object" or color_value.dtype.name == "category"
+ is_categorical = is_text_like(color_value, strict_object=False)
if not is_categorical:
return color_value.astype(float), False, None
diff --git a/shapash/utils/custom_thread.py b/shapash/utils/custom_thread.py
index 719237e4..a4c96898 100644
--- a/shapash/utils/custom_thread.py
+++ b/shapash/utils/custom_thread.py
@@ -4,6 +4,7 @@
import sys
import threading
+from collections.abc import Callable
class CustomThread(threading.Thread):
@@ -14,12 +15,17 @@ class CustomThread(threading.Thread):
----------
threading : threading.Thread
Thread which you want to instanciate
+ on_kill : Callable, optional
+ Extra callback invoked when the thread is killed, in addition to
+ stopping the traced run loop (e.g. to shut down a server bound to
+ this thread).
"""
- def __init__(self, *args, **keywords):
+ def __init__(self, *args, on_kill: Callable[[], None] | None = None, **keywords):
threading.Thread.__init__(self, *args, **keywords)
self.killed = False
self.__run_backup = None
+ self.on_kill = on_kill
def start(self):
"""Starts the thread"""
@@ -54,4 +60,6 @@ def kill(self):
"""
Kill the current Thread
"""
+ if self.on_kill is not None:
+ self.on_kill()
self.killed = True
diff --git a/shapash/utils/dtypes.py b/shapash/utils/dtypes.py
new file mode 100644
index 00000000..8e94d27e
--- /dev/null
+++ b/shapash/utils/dtypes.py
@@ -0,0 +1,32 @@
+import pandas as pd
+from pandas.api.types import is_string_dtype
+
+
+def is_text_like(series: pd.Series, strict_object: bool = False) -> bool:
+ """Return whether a series should be treated as text-like/categorical.
+
+ Parameters
+ ----------
+ series : pd.Series
+ Series to evaluate.
+ strict_object : bool, default=False
+ If True, ``object`` dtype is accepted only when inferred values are
+ textual or empty. If False, all ``object`` dtype columns are accepted.
+ """
+ dtype = series.dtype
+
+ if isinstance(dtype, pd.CategoricalDtype):
+ return True
+
+ if dtype.name == "object":
+ if not strict_object:
+ return True
+ inferred_dtype = pd.api.types.infer_dtype(series, skipna=True)
+ return inferred_dtype in ("string", "empty")
+
+ return is_string_dtype(dtype)
+
+
+def text_like_columns(df: pd.DataFrame, strict_object: bool = False) -> list[str]:
+ """Return dataframe columns considered text-like by ``is_text_like``."""
+ return [col for col in df.columns if is_text_like(df[col], strict_object=strict_object)]
diff --git a/shapash/utils/transform.py b/shapash/utils/transform.py
index 19bfabff..8b1c0912 100644
--- a/shapash/utils/transform.py
+++ b/shapash/utils/transform.py
@@ -21,6 +21,7 @@
supported_sklearn,
transform_ct,
)
+from shapash.utils.dtypes import text_like_columns
# TODO
# encode targeted variable ? from sklearn.preprocessing import LabelEncoder
@@ -391,7 +392,13 @@ def handle_categorical_missing(df: pd.DataFrame) -> pd.DataFrame:
df : pd.DataFrame
Pandas dataframe on which we will replace the missing values
"""
- categorical_cols = df.select_dtypes(include=["object"]).columns
+ categorical_cols = text_like_columns(df, strict_object=False)
df_handle_missing = df.copy()
+
+ categorical_dtype_cols = df_handle_missing.select_dtypes(include=["category"]).columns
+ for col in categorical_dtype_cols:
+ if "missing" not in df_handle_missing[col].cat.categories:
+ df_handle_missing[col] = df_handle_missing[col].cat.add_categories(["missing"])
+
df_handle_missing[categorical_cols] = df_handle_missing[categorical_cols].fillna("missing")
return df_handle_missing
diff --git a/shapash/utils/utils.py b/shapash/utils/utils.py
index 09037bba..2f51c5f8 100644
--- a/shapash/utils/utils.py
+++ b/shapash/utils/utils.py
@@ -199,6 +199,34 @@ def truncate_str(text, maxlen=40):
return text
+MISSING_VALUE_DISPLAY = "missing"
+
+
+def format_missing_value(value, missing_display=MISSING_VALUE_DISPLAY):
+ """
+ return a unified display value for null entries
+
+ Parameters
+ ----------
+ value : any
+ value to display, can be null (NaN, None, NaT, pd.NA)
+ missing_display : str
+ text displayed in place of null values
+
+ Returns
+ -------
+ any
+ missing_display if the value is null, the original value otherwise
+ """
+ try:
+ if pd.isna(value):
+ return missing_display
+ except (TypeError, ValueError):
+ # non-scalar values (list, array, ...) are kept unchanged
+ pass
+ return value
+
+
def compute_digit_number(value, significant_digits: int = 4):
"""
return int, number of digits to display
@@ -352,7 +380,7 @@ def compute_top_correlations_features(corr: pd.DataFrame, max_features: int) ->
list
"""
sorted_corr = corr.abs().unstack().sort_values(kind="quicksort")[::-1]
- set_features = set()
+ set_features: set = set()
i = 0
while len(set_features) < max_features and i < len(sorted_corr):
if sorted_corr.index[i][0] != sorted_corr.index[i][1]:
diff --git a/shapash/webapp/smart_app.py b/shapash/webapp/smart_app.py
index 68454a64..d3ce6a7f 100644
--- a/shapash/webapp/smart_app.py
+++ b/shapash/webapp/smart_app.py
@@ -7,6 +7,7 @@
import random
import re
from math import isfinite, log10
+from typing import Any
import dash
import dash_bootstrap_components as dbc
@@ -46,7 +47,7 @@
)
from shapash.webapp.utils.explanations import Explanations
from shapash.webapp.utils.MyGraph import MyGraph
-from shapash.webapp.utils.utils import check_row, get_index_type, round_to_k
+from shapash.webapp.utils.utils import check_row, get_datatable_data_and_tooltips, get_index_type, round_to_k
def _create_input_modal(component_id, label, tooltip):
@@ -69,7 +70,7 @@ class SmartApp:
SmartExplainer instance to point to.
"""
- def __init__(self, explainer, settings: dict = None):
+ def __init__(self, explainer, settings: dict | None = None):
"""
Init on class instantiation, everything to be able to run the app on server.
Parameters
@@ -129,7 +130,7 @@ def __init__(self, explainer, settings: dict = None):
self.label = None
self.selected_feature = self.explainer.features_imp.idxmax()
self.max_threshold = self.explainer.contributions.map(lambda x: round_to_k(x, k=1)).max().max()
- self.list_index = []
+ self.list_index: list = []
self.subset = None
self.last_click_data = None
@@ -141,11 +142,17 @@ def __init__(self, explainer, settings: dict = None):
self.init_data()
# COMPONENTS
- self.components = {"menu": {}, "table": {}, "graph": {}, "filter": {}, "settings": {}}
+ self.components: dict[str, dict[str, Any]] = {
+ "menu": {},
+ "table": {},
+ "graph": {},
+ "filter": {},
+ "settings": {},
+ }
self.init_components()
# LAYOUT
- self.skeleton = {"navbar": {}, "body": {}}
+ self.skeleton: dict[str, Any] = {"navbar": {}, "body": {}}
self.make_skeleton()
self.app.layout = html.Div([self.skeleton["navbar"], self.skeleton["body"]])
@@ -424,13 +431,11 @@ def init_components(self):
self.adjust_menu()
+ table_data, table_tooltip_data = get_datatable_data_and_tooltips(self.round_dataframe, self.dataframe)
self.components["table"]["dataset"] = dash_table.DataTable(
id="dataset",
- data=self.round_dataframe.to_dict("records"),
- tooltip_data=[
- {column: {"value": str(value), "type": "text"} for column, value in row.items()}
- for row in self.dataframe.to_dict("index").values()
- ],
+ data=table_data,
+ tooltip_data=table_tooltip_data,
tooltip_duration=2000,
columns=[{"name": i, "id": i} for i in self.dataframe.columns],
tooltip_header={
@@ -2101,11 +2106,7 @@ def update_datatable(
df = self.round_dataframe
else:
raise dash.exceptions.PreventUpdate
- data = df.to_dict("records")
- tooltip_data = [
- {column: {"value": str(value), "type": "text"} for column, value in row.items()}
- for row in df.to_dict("index").values()
- ]
+ data, tooltip_data = get_datatable_data_and_tooltips(df)
return (
data,
tooltip_data,
diff --git a/shapash/webapp/utils/callbacks.py b/shapash/webapp/utils/callbacks.py
index 7a7ffaa5..0ad1108f 100644
--- a/shapash/webapp/utils/callbacks.py
+++ b/shapash/webapp/utils/callbacks.py
@@ -11,6 +11,7 @@
from dash.exceptions import PreventUpdate
from plotly.graph_objs import Figure
+from shapash.utils.utils import format_missing_value
from shapash.webapp.utils.MyGraph import MyGraph
@@ -269,7 +270,7 @@ def get_indexes_from_datatable(data: list, list_index: list | None = None) -> li
"""
indexes = [d["_index_"] for d in data]
if list_index is not None and (len(indexes) == len(list_index) or len(indexes) == 0):
- indexes = None
+ return None
return indexes
@@ -399,7 +400,7 @@ def get_id_card_features(data: list, selected: int, special_cols: list, features
def get_id_card_contrib(
- data: dict, index: int, features_dict: dict, columns_dict: dict, label_num: int = None
+ data: dict, index: int, features_dict: dict, columns_dict: dict, label_num: int | None = None
) -> pd.DataFrame:
"""Get the contributions of the selected index for the identity card.
@@ -506,7 +507,7 @@ def create_id_card_layout(selected_data: pd.DataFrame, additional_features_dict:
dbc.Row(
[
dbc.Col(dbc.Label(row["feature_name"]), width=3, style=label_style),
- dbc.Col(dbc.Label(row["feature_value"]), width=5, className="id_card_solid"),
+ dbc.Col(dbc.Label(format_missing_value(row["feature_value"])), width=5, className="id_card_solid"),
dbc.Col(width=1),
(
dbc.Col(
@@ -642,9 +643,10 @@ def create_filter_modalities_selection(value: str, filter_id: dict, round_datafr
_first = _non_null.iloc[0] if len(_non_null) > 0 else None
if type(_first) is np.bool_ or type(_first) is bool:
+ radio_options: list[dcc.RadioItems.Options] = [{"label": str(val), "value": val} for val in _non_null.unique()]
new_element = html.Div(
dcc.RadioItems(
- [{"label": str(val), "value": val} for val in _non_null.unique()],
+ radio_options,
id={"type": "dynamic-bool", "index": filter_id["index"]},
value=_first,
inline=False,
@@ -655,10 +657,11 @@ def create_filter_modalities_selection(value: str, filter_id: dict, round_datafr
# If feature has integer type with at most 20 values or string type (no limits on number of values),
# then display Dropdown component
# Notice that integer column with NaN values is considered as float, so it will not be displayed as Dropdown.
+ dropdown_options: list[dcc.Dropdown.Options] = [{"label": i, "value": i} for i in np.sort(_non_null.unique())]
new_element = html.Div(
dcc.Dropdown(
id={"type": "dynamic-str", "index": filter_id["index"]},
- options=[{"label": i, "value": i} for i in np.sort(_non_null.unique())],
+ options=dropdown_options,
multi=True,
),
style={"width": "65%", "margin-left": "20px"},
@@ -701,17 +704,19 @@ def create_filter_modalities_selection(value: str, filter_id: dict, round_datafr
return new_element
-def handle_page_navigation(triggered_input: str, page: int | str, selected_feature: str) -> tuple[int, str]:
+def handle_page_navigation(
+ triggered_input: str, page: int | str, selected_feature: str | None
+) -> tuple[int, str | None]:
"""
Handle the navigation between different pages based on user input.
Args:
triggered_input (str): The input that triggered the navigation.
page (Union[int, str]): The current page number.
- selected_feature (str): The currently selected feature.
+ selected_feature (Optional[str]): The currently selected feature.
Returns:
- tuple[int, str]: Updated page number and selected feature.
+ tuple[int, Optional[str]]: Updated page number and selected feature.
"""
page = int(page)
if triggered_input == "page_left.n_clicks":
@@ -746,7 +751,7 @@ def update_click_data_on_subset_changes_if_needed(click_data: dict, triggered_in
return click_data
-def get_selected_feature(click_data: dict, inv_features_dict: dict) -> str:
+def get_selected_feature(click_data: dict, inv_features_dict: dict) -> str | None:
"""
Retrieve the selected feature from the click data.
@@ -755,7 +760,7 @@ def get_selected_feature(click_data: dict, inv_features_dict: dict) -> str:
inv_features_dict (dict): Dictionary mapping feature IDs to feature names.
Returns:
- str: The selected feature, if any.
+ Optional[str]: The selected feature, if any.
"""
return inv_features_dict.get(get_feature_from_clicked_data(click_data)) if click_data else None
@@ -763,28 +768,29 @@ def get_selected_feature(click_data: dict, inv_features_dict: dict) -> str:
def handle_group_display_logic(
bool_group: bool,
triggered_input: str,
- selected_feature: str,
+ selected_feature: str | None,
selected_click_data,
- click_data: dict,
+ click_data: dict | None,
click_data_store: dict,
selected_click_data_store,
features_groups: dict,
features_dict: dict,
-) -> tuple[str, str, dict]:
+) -> tuple[str | None, str | None, dict | None, object]:
"""
Handle the display logic for feature groups.
Args:
bool_group (bool): Whether to display feature groups.
triggered_input (str): The input that triggered the update.
- selected_feature (str): The currently selected feature.
- click_data (dict): The current click data.
+ selected_feature (Optional[str]): The currently selected feature.
+ click_data (Optional[dict]): The current click data.
click_data_store (dict): Stored click data.
features_groups (dict): Dictionary of feature groups.
features_dict (dict): Dictionary of features.
Returns:
- tuple[str, str, dict]: Updated selected feature, group name, and click data.
+ tuple[Optional[str], Optional[str], Optional[dict], object]: Updated selected feature,
+ group name, click data, and selected click data.
"""
group_name = None
selected_feature_group = None
@@ -823,7 +829,7 @@ def handle_group_display_logic(
def determine_total_pages_and_display(
explainer: "SmartExplainer", features: int, bool_group: bool, group_name: str, page: int
-) -> tuple[int, str, int]:
+) -> tuple[int, dict[str, str], int]:
"""
Determine the total number of pages and the display properties.
@@ -835,7 +841,7 @@ def determine_total_pages_and_display(
page (int): Current page number.
Returns:
- tuple[int, str, int]: Total pages, display properties, and updated page number.
+ tuple[int, dict[str, str], int]: Total pages, display properties, and updated page number.
"""
display_groups = explainer.features_groups is not None and bool_group
if explainer._case == "classification":
diff --git a/shapash/webapp/utils/utils.py b/shapash/webapp/utils/utils.py
index 27675e3a..8488728e 100644
--- a/shapash/webapp/utils/utils.py
+++ b/shapash/webapp/utils/utils.py
@@ -1,6 +1,35 @@
import pandas as pd
from pandas.api.types import is_any_real_numeric_dtype
+from shapash.utils.utils import format_missing_value
+
+
+def get_datatable_data_and_tooltips(data_df, tooltip_df=None):
+ """
+ Build the data records and tooltips of the dataset DataTable,
+ with a unified display of missing values.
+
+ Parameters
+ ----------
+ data_df : pd.DataFrame
+ Dataframe used for the cells of the datatable
+ tooltip_df : pd.DataFrame (optional)
+ Dataframe used for the tooltips of the datatable, data_df if not provided
+
+ Returns
+ -------
+ tuple
+ data records and tooltip_data of the datatable
+ """
+ if tooltip_df is None:
+ tooltip_df = data_df
+ data = [{col: format_missing_value(val) for col, val in row.items()} for row in data_df.to_dict("records")]
+ tooltip_data = [
+ {col: {"value": str(format_missing_value(val)), "type": "text"} for col, val in row.items()}
+ for row in tooltip_df.to_dict("records")
+ ]
+ return data, tooltip_data
+
def round_to_k(x, k):
"""
diff --git a/shapash/webapp/webapp_launch.py b/shapash/webapp/webapp_launch.py
index dbd3c267..2e770016 100644
--- a/shapash/webapp/webapp_launch.py
+++ b/shapash/webapp/webapp_launch.py
@@ -93,6 +93,8 @@
},
}
+additional_features_dict: dict | None = None
+
if CASE == 1:
features = ["Pclass", "Survived", "Embarked", "Sex", "Age", "SibSp", "Parch"]
for col in list(feature_dict.keys()):
diff --git a/shapash/webapp/webapp_launch_DVF.py b/shapash/webapp/webapp_launch_DVF.py
index c21b7a22..7b260caa 100644
--- a/shapash/webapp/webapp_launch_DVF.py
+++ b/shapash/webapp/webapp_launch_DVF.py
@@ -11,14 +11,15 @@
from shapash import SmartExplainer
from shapash.data.data_loader import data_loading
from shapash.explainer.smart_explainer import DEFAULT_HOST
+from shapash.utils.dtypes import text_like_columns
house_df, house_dict = data_loading("house_prices")
y_df = house_df["SalePrice"].to_frame()
X_df = house_df[house_df.columns.difference(["SalePrice"])]
house_df.head()
-categorical_features = [col for col in X_df.columns if X_df[col].dtype == "object"]
-encoder = OrdinalEncoder(cols=categorical_features, handle_unknown="ignore", return_df=True).fit(X_df)
+categorical_features = text_like_columns(X_df, strict_object=False)
+encoder = OrdinalEncoder(cols=categorical_features, handle_unknown="return_nan", return_df=True).fit(X_df)
X_df = encoder.transform(X_df)
Xtrain, Xtest, ytrain, ytest = train_test_split(X_df, y_df, train_size=0.75, random_state=1)
diff --git a/tests/integration_tests/test_contributions_multiclass.py b/tests/integration_tests/test_contributions_multiclass.py
index ad1d68a7..dee68f39 100644
--- a/tests/integration_tests/test_contributions_multiclass.py
+++ b/tests/integration_tests/test_contributions_multiclass.py
@@ -83,7 +83,7 @@ def get_predictions(self, model, **args):
[type]
[description]
"""
- model.fit(self.x_train, self.y_train)
+ model.fit(self.x_train, self.y_train.values.ravel())
if args:
return model.predict(self.x_test, **args)
else:
@@ -94,7 +94,7 @@ def test_rank_contributions_1(self):
Unit test rank contributions 1
"""
model = RandomForestClassifier(n_estimators=3)
- model.fit(self.x_train, self.y_train)
+ model.fit(self.x_train, self.y_train.values.ravel())
explainer = shap.TreeExplainer(model)
shap_values = explainer.shap_values(self.x_test)
slist = [
diff --git a/tests/integration_tests/test_report_generation.py b/tests/integration_tests/test_report_generation.py
index 5c54aa2e..caf39504 100644
--- a/tests/integration_tests/test_report_generation.py
+++ b/tests/integration_tests/test_report_generation.py
@@ -24,7 +24,7 @@ def setUp(self):
df["x3"] = np.random.choice(["A", "B", "C", "D"], df.shape[0])
df["x4"] = np.random.choice(["A", "B", "C", np.nan], df.shape[0])
df = df.set_index("id")
- encoder = ce.OrdinalEncoder(cols=["x3", "x4"], handle_unknown="None")
+ encoder = ce.OrdinalEncoder(cols=["x3", "x4"], handle_unknown="return_nan")
encoder_fitted = encoder.fit(df)
df_encoded = encoder_fitted.transform(df)
clf = cb.CatBoostClassifier(n_estimators=1).fit(df_encoded[["x1", "x2", "x3", "x4"]], df_encoded["y"])
diff --git a/tests/unit_tests/explainer/test_smart_explainer.py b/tests/unit_tests/explainer/test_smart_explainer.py
index 9fbbe14f..5d1bd1be 100644
--- a/tests/unit_tests/explainer/test_smart_explainer.py
+++ b/tests/unit_tests/explainer/test_smart_explainer.py
@@ -203,7 +203,7 @@ def test_compile_2(self):
df["x1"] = np.random.randint(1, 123, df.shape[0])
df["x2"] = ["S", "M", "S", "D", "M"]
df = df.set_index("id")
- encoder = ce.OrdinalEncoder(cols=["x2"], handle_unknown="None")
+ encoder = ce.OrdinalEncoder(cols=["x2"], handle_unknown="return_nan")
encoder_fitted = encoder.fit(df)
df_encoded = encoder_fitted.transform(df)
output = df[["x1", "x2"]].copy()
@@ -1005,7 +1005,7 @@ def test_to_smartpredictor_1(self):
df["x1"] = np.random.randint(1, 123, df.shape[0])
df["x2"] = ["S", "M", "S", "D", "M"]
df = df.set_index("id")
- encoder = ce.OrdinalEncoder(cols=["x2"], handle_unknown="None")
+ encoder = ce.OrdinalEncoder(cols=["x2"], handle_unknown="return_nan")
encoder_fitted = encoder.fit(df[["x1", "x2"]])
df_encoded = encoder_fitted.transform(df[["x1", "x2"]])
clf = cb.CatBoostClassifier(n_estimators=1).fit(df_encoded[["x1", "x2"]], df["y"])
diff --git a/tests/unit_tests/explainer/test_smart_plotter.py b/tests/unit_tests/explainer/test_smart_plotter.py
index 312cf5ae..acbb5d89 100644
--- a/tests/unit_tests/explainer/test_smart_plotter.py
+++ b/tests/unit_tests/explainer/test_smart_plotter.py
@@ -19,6 +19,7 @@
from shapash.explainer.multi_decorator import MultiDecorator
from shapash.explainer.smart_state import SmartState
from shapash.plots.plot_bar_chart import plot_bar_chart
+from shapash.plots.plot_contribution import plot_scatter
from shapash.plots.plot_evaluation_metrics import plot_confusion_matrix
from shapash.plots.plot_feature_importance import _plot_features_import
from shapash.plots.plot_line_comparison import plot_line_comparison
@@ -87,7 +88,7 @@ def setUp(self):
index=["person_A", "person_B"],
)
self.features_compacity = {"features_needed": [1, 1], "distance_reached": np.array([0.12, 0.16])}
- encoder = ce.OrdinalEncoder(cols=["X1"], handle_unknown="None").fit(self.x_init)
+ encoder = ce.OrdinalEncoder(cols=["X1"], handle_unknown="return_nan").fit(self.x_init)
model = CatBoostClassifier().fit(encoder.transform(self.x_init), [0, 1])
self.model = model
# Declare explainer object
@@ -1190,6 +1191,71 @@ def test_contribution_plot_nan_numeric_violin(self):
ticktext = list(output.layout.xaxis.ticktext) if output.layout.xaxis.ticktext else []
assert "missing" in ticktext
+ # the hover text of the scatter points of the "missing" modality must also show "missing"
+ missing_scatters = [
+ t for t in output.data if t.type == "scatter" and t.mode == "markers" and t.name == "missing"
+ ]
+ assert len(missing_scatters) > 0
+ for trace in missing_scatters:
+ assert all(row[0] == "missing" for row in trace.customdata)
+
+ def test_contribution_plot_nan_object_scatter(self):
+ """
+ Object feature with null values must render on the scatter contribution plot
+ as an explicit "missing" modality with an "x" marker symbol, with "missing"
+ surfaced in the hover customdata.
+ Regression test for https://github.com/MAIF/shapash/issues/721
+ """
+ n_val, n_nan = 8, 2
+ feature_values = pd.DataFrame(
+ {"str_feat": ["a", "b", None, "c", "b", None, "d", "e"]}, index=list(range(n_val))
+ )
+ contributions = pd.DataFrame({"str_feat": [0.1, -0.2, 0.3, 0.4, -0.1, 0.2, 0.15, -0.3]})
+ output = plot_scatter(
+ feature_values,
+ contributions,
+ "str_feat",
+ "regression",
+ self.smart_explainer.plot._style_dict,
+ )
+
+ marker_traces = [t for t in output.data if t.type == "scatter" and t.mode == "markers"]
+ assert len(marker_traces) == 1
+ trace = marker_traces[0]
+
+ x_arr = list(trace.x)
+ assert len(x_arr) == n_val
+ assert all(isinstance(x, str) for x in x_arr)
+ assert x_arr.count("missing") == n_nan
+
+ symbols = list(trace.marker.symbol)
+ assert symbols.count("x") == n_nan
+ assert symbols.count("circle") == n_val - n_nan
+
+ customdata_col0 = [row[0] for row in trace.customdata]
+ nan_customdata = [v for v, s in zip(customdata_col0, symbols) if s == "x"]
+ assert all(v == "missing" for v in nan_customdata)
+
+ def test_plot_bar_chart_nan_display(self):
+ """
+ Null feature values must be displayed as "missing" in the local plot
+ y-axis labels and hover text.
+ Regression test for https://github.com/MAIF/shapash/issues/721
+ """
+ var_dict = ["X1", "X2"]
+ x_val = [np.nan, "PhD"]
+ contributions = [-3.4, 0.78]
+ self.smart_explainer._case = "regression"
+ fig_output = plot_bar_chart("ind", var_dict, x_val, contributions, self.smart_explainer.plot._style_dict)
+
+ ylabels = [bar.y[0] for bar in fig_output.data]
+ assert "X1 :
missing" in ylabels
+ assert not any("nan" in str(label) for label in ylabels)
+
+ hoverlabels = [bar.customdata[0] for bar in fig_output.data]
+ assert any("missing" in label for label in hoverlabels)
+ assert not any("nan" in str(label) for label in hoverlabels)
+
def test_plot_features_import_1(self):
"""
Unit test plot features import 1
diff --git a/tests/unit_tests/explainer/test_smart_predictor.py b/tests/unit_tests/explainer/test_smart_predictor.py
index 3c84a73d..9300a7f5 100644
--- a/tests/unit_tests/explainer/test_smart_predictor.py
+++ b/tests/unit_tests/explainer/test_smart_predictor.py
@@ -57,7 +57,7 @@ def setUp(self):
df["x1"] = np.random.randint(1, 123, df.shape[0])
df["x2"] = ["S", "M", "S", "D", "M"]
df = df.set_index("id")
- encoder = ce.OrdinalEncoder(cols=["x2"], handle_unknown="None")
+ encoder = ce.OrdinalEncoder(cols=["x2"], handle_unknown="return_nan")
encoder_fitted = encoder.fit(df[["x1", "x2"]])
df_encoded = encoder_fitted.transform(df[["x1", "x2"]])
clf = cb.CatBoostClassifier(n_estimators=1).fit(df_encoded[["x1", "x2"]], df["y"])
@@ -101,7 +101,7 @@ def setUp(self):
)
self.predictor_1.backend.state = SmartState()
df["x2"] = np.random.randint(1, 100, df.shape[0])
- encoder = ce.OrdinalEncoder(cols=["x2"], handle_unknown="None")
+ encoder = ce.OrdinalEncoder(cols=["x2"], handle_unknown="return_nan")
encoder_fitted = encoder.fit(df[["x1", "x2"]])
df_encoded = encoder_fitted.transform(df[["x1", "x2"]])
diff --git a/tests/unit_tests/report/test_common.py b/tests/unit_tests/report/test_common.py
index e8039b6a..aab01340 100644
--- a/tests/unit_tests/report/test_common.py
+++ b/tests/unit_tests/report/test_common.py
@@ -22,6 +22,38 @@ def test_series_dtype_1(self):
assert series_dtype(s) == VarType.TYPE_CAT
+ def test_series_dtype_1_object_strings_with_nan(self):
+ """
+ Test object string series with missing values
+ """
+ s = pd.Series(["a", None, "b"], dtype=object)
+
+ assert series_dtype(s) == VarType.TYPE_CAT
+
+ def test_series_dtype_1_object_non_string(self):
+ """
+ Test object series with non-string values
+ """
+ s = pd.Series([["a"], {"b": 1}], dtype=object)
+
+ assert series_dtype(s) == VarType.TYPE_UNSUPPORTED
+
+ def test_series_dtype_1_object_datetime_is_unsupported(self):
+ """
+ Test object datetime-like series is unsupported.
+ """
+ s = pd.Series([pd.Timestamp("2024-01-01"), pd.Timestamp("2024-01-02")], dtype=object)
+
+ assert series_dtype(s) == VarType.TYPE_UNSUPPORTED
+
+ def test_series_dtype_1_object_empty_is_categorical(self):
+ """
+ Test empty object series remains categorical.
+ """
+ s = pd.Series([None, None], dtype=object)
+
+ assert series_dtype(s) == VarType.TYPE_CAT
+
def test_series_dtype_2(self):
"""
Test bool series
diff --git a/tests/unit_tests/report/test_plots.py b/tests/unit_tests/report/test_plots.py
index 213579ed..b3d46dd9 100644
--- a/tests/unit_tests/report/test_plots.py
+++ b/tests/unit_tests/report/test_plots.py
@@ -88,6 +88,19 @@ def test_plot_categorical_distribution_1(self):
assert fig.data[1].type == "bar"
assert len(fig.data[0]['x']) == 2
+ def test_plot_categorical_distribution_no_hue(self):
+ """
+ Test that the plot can be generated without a hue/col_splitter column.
+ """
+ df = pd.DataFrame({"int_data": [0, 0, 0, 1, 1, 0]})
+
+ fig = plot_categorical_distribution(df, "int_data")
+
+ assert isinstance(fig, go.Figure)
+ assert len(fig.data) == 1
+ assert fig.data[0].type == "bar"
+ assert len(fig.data[0]["x"]) == 2
+
def test_plot_categorical_distribution_2(self):
df = pd.DataFrame(
{"int_data": [0, 0, 0, 1, 1, 0], "data_train_test": ["train", "train", "train", "train", "train", "train"]}
diff --git a/tests/unit_tests/report/test_project_report.py b/tests/unit_tests/report/test_project_report.py
index 4a66020f..89f1197a 100644
--- a/tests/unit_tests/report/test_project_report.py
+++ b/tests/unit_tests/report/test_project_report.py
@@ -162,7 +162,7 @@ def test_display_dataset_analysis_3(self):
df = self.df.copy()
df["x1"] = "a"
df["x2"] = df["x2"].astype(str)
- encoder = OrdinalEncoder(cols=["x1", "x2"], handle_unknown="ignore", return_df=True).fit(df)
+ encoder = OrdinalEncoder(cols=["x1", "x2"], handle_unknown="return_nan", return_df=True).fit(df)
df = encoder.transform(df)
diff --git a/tests/unit_tests/utils/test_category_encoders_backend.py b/tests/unit_tests/utils/test_category_encoders_backend.py
index ad023abc..ac648f1d 100644
--- a/tests/unit_tests/utils/test_category_encoders_backend.py
+++ b/tests/unit_tests/utils/test_category_encoders_backend.py
@@ -11,10 +11,21 @@
import xgboost
from sklearn.ensemble import GradientBoostingClassifier
+from shapash.utils.category_encoder_backend import inv_transform_ordinal
from shapash.utils.transform import apply_preprocessing, get_col_mapping_ce, inverse_transform
class TestInverseTransformCaterogyEncoder(unittest.TestCase):
+ def test_inv_transform_ordinal_string_dtype_preserves_missing(self):
+ """Check pandas string dtype keeps missing values as on inverse transform."""
+ x_in = pd.DataFrame({"city": [1, 2, 3]})
+ encoding = [{"col": "city", "mapping": {"A": 1, "B": 2}, "data_type": "string"}]
+
+ result = inv_transform_ordinal(x_in, encoding)
+
+ expected = pd.DataFrame({"city": pd.Series(["A", "B", pd.NA], dtype="string")})
+ pd.testing.assert_frame_equal(result, expected)
+
def test_inverse_transform_1(self):
"""
Test no preprocessing
diff --git a/tests/unit_tests/utils/test_check.py b/tests/unit_tests/utils/test_check.py
index 8cb718e5..1396c36b 100644
--- a/tests/unit_tests/utils/test_check.py
+++ b/tests/unit_tests/utils/test_check.py
@@ -575,6 +575,15 @@ def test_check_postprocessing_1(self):
check_postprocessing(features_types, postprocessing5)
check_postprocessing(features_types, postprocessing6)
+ def test_check_postprocessing_accepts_pandas_string_metadata(self):
+ """Unit test pandas string dtype metadata validation for postprocessing."""
+ features_types = {"Col1": "string[python]", "Col2": "string[pyarrow]"}
+ case_postprocessing = {"Col1": {"type": "case", "rule": "lower"}}
+ regex_postprocessing = {"Col2": {"type": "regex", "rule": {"in": "A", "out": "a"}}}
+
+ check_postprocessing(features_types, case_postprocessing)
+ check_postprocessing(features_types, regex_postprocessing)
+
def test_check_preprocessing_options_1(self):
"""
Unit test check_preprocessing_options 1
diff --git a/tests/unit_tests/utils/test_dtypes.py b/tests/unit_tests/utils/test_dtypes.py
new file mode 100644
index 00000000..5d5dcc3f
--- /dev/null
+++ b/tests/unit_tests/utils/test_dtypes.py
@@ -0,0 +1,35 @@
+import pandas as pd
+
+from shapash.utils.dtypes import is_text_like, text_like_columns
+
+
+def test_is_text_like_object_string_is_true_in_strict_mode():
+ s = pd.Series(["a", None, "b"], dtype=object)
+
+ assert is_text_like(s, strict_object=True) is True
+
+
+def test_is_text_like_object_mixed_is_false_in_strict_mode():
+ s = pd.Series([["a"], {"b": 1}], dtype=object)
+
+ assert is_text_like(s, strict_object=True) is False
+
+
+def test_is_text_like_object_mixed_is_true_in_permissive_mode():
+ s = pd.Series([["a"], {"b": 1}], dtype=object)
+
+ assert is_text_like(s, strict_object=False) is True
+
+
+def test_text_like_columns_strict_and_permissive_modes():
+ df = pd.DataFrame(
+ {
+ "txt": pd.Series(["x", None], dtype=object),
+ "mixed": pd.Series([["a"], {"b": 1}], dtype=object),
+ "cat": pd.Series(["a", "b"], dtype="category"),
+ "num": [1, 2],
+ }
+ )
+
+ assert text_like_columns(df, strict_object=True) == ["txt", "cat"]
+ assert text_like_columns(df, strict_object=False) == ["txt", "mixed", "cat"]
diff --git a/tests/unit_tests/utils/test_transform.py b/tests/unit_tests/utils/test_transform.py
index 15376cce..77da39d4 100644
--- a/tests/unit_tests/utils/test_transform.py
+++ b/tests/unit_tests/utils/test_transform.py
@@ -270,3 +270,48 @@ def test_handle_categorical_missing(self):
)
assert_frame_equal(df_test, df_expected)
+
+ def test_handle_categorical_missing_string_dtype(self):
+ """Fill pd.NA for pandas StringDtype columns."""
+ df_test = pd.DataFrame(
+ {
+ "city": pd.Series([pd.NA, "paris", "chicago"], dtype="string"),
+ "state": ["US", "FR", "FR"],
+ }
+ )
+
+ df_result = handle_categorical_missing(df_test)
+
+ df_expected = pd.DataFrame(
+ {
+ "city": pd.Series(["missing", "paris", "chicago"], dtype="string"),
+ "state": ["US", "FR", "FR"],
+ }
+ )
+
+ assert_frame_equal(df_result, df_expected)
+
+ def test_handle_categorical_missing_category_dtype(self):
+ """Add 'missing' category and fill NaN for category columns."""
+ df_test = pd.DataFrame(
+ {
+ "city": pd.Series(
+ pd.Categorical([np.nan, "paris", "chicago"], categories=["paris", "chicago"])
+ ),
+ "state": ["US", "FR", "FR"],
+ }
+ )
+
+ df_result = handle_categorical_missing(df_test)
+
+ self.assertEqual(df_result["city"].dtype.name, "category")
+ self.assertIn("missing", df_result["city"].cat.categories)
+ self.assertFalse(df_result["city"].isna().any())
+
+ expected_city = pd.Series(
+ pd.Categorical(
+ ["missing", "paris", "chicago"],
+ categories=["paris", "chicago", "missing"],
+ )
+ )
+ pd.testing.assert_series_equal(df_result["city"], expected_city, check_names=False)
diff --git a/tests/unit_tests/utils/test_utils.py b/tests/unit_tests/utils/test_utils.py
index 3b0d70da..62122444 100644
--- a/tests/unit_tests/utils/test_utils.py
+++ b/tests/unit_tests/utils/test_utils.py
@@ -4,10 +4,12 @@
import pandas as pd
from shapash.utils.utils import (
+ MISSING_VALUE_DISPLAY,
add_line_break,
compute_digit_number,
compute_sorted_variables_interactions_list_indices,
compute_top_correlations_features,
+ format_missing_value,
inclusion,
is_nested_list,
maximum_difference_sort_value,
@@ -160,3 +162,34 @@ def test_compute_top_correlations_features_2(self):
list_features = compute_top_correlations_features(corr=corr, max_features=5)
assert len(list_features) == 5
+
+ def test_format_missing_value_1(self):
+ """
+ Test null values are unified to the missing display value
+ """
+ assert format_missing_value(np.nan) == MISSING_VALUE_DISPLAY
+ assert format_missing_value(None) == MISSING_VALUE_DISPLAY
+ assert format_missing_value(pd.NA) == MISSING_VALUE_DISPLAY
+ assert format_missing_value(pd.NaT) == MISSING_VALUE_DISPLAY
+
+ def test_format_missing_value_2(self):
+ """
+ Test non-null values are kept unchanged
+ """
+ assert format_missing_value("") == ""
+ assert format_missing_value(0) == 0
+ assert format_missing_value(3.2) == 3.2
+ assert format_missing_value("abc") == "abc"
+
+ def test_format_missing_value_3(self):
+ """
+ Test non-scalar values are kept unchanged
+ """
+ value = [1, np.nan]
+ assert format_missing_value(value) is value
+
+ def test_format_missing_value_4(self):
+ """
+ Test custom missing display value
+ """
+ assert format_missing_value(np.nan, missing_display="N/A") == "N/A"
diff --git a/tests/unit_tests/webapp/utils/test_utils.py b/tests/unit_tests/webapp/utils/test_utils.py
index 4f75d6a0..4b694c37 100644
--- a/tests/unit_tests/webapp/utils/test_utils.py
+++ b/tests/unit_tests/webapp/utils/test_utils.py
@@ -1,9 +1,43 @@
import unittest
-from shapash.webapp.utils.utils import round_to_k
+import numpy as np
+import pandas as pd
+
+from shapash.webapp.utils.utils import get_datatable_data_and_tooltips, round_to_k
class TestUtils(unittest.TestCase):
+ def test_get_datatable_data_and_tooltips_1(self):
+ """
+ Null values must be displayed as "missing" in the datatable cells and tooltips
+ """
+ df = pd.DataFrame({"num": [1.5, np.nan], "txt": ["a", None]})
+ data, tooltip_data = get_datatable_data_and_tooltips(df)
+
+ assert data[0] == {"num": 1.5, "txt": "a"}
+ assert data[1] == {"num": "missing", "txt": "missing"}
+ assert tooltip_data[0] == {
+ "num": {"value": "1.5", "type": "text"},
+ "txt": {"value": "a", "type": "text"},
+ }
+ assert tooltip_data[1] == {
+ "num": {"value": "missing", "type": "text"},
+ "txt": {"value": "missing", "type": "text"},
+ }
+
+ def test_get_datatable_data_and_tooltips_2(self):
+ """
+ Tooltips can be built from a different (unrounded) dataframe than the cells
+ """
+ data_df = pd.DataFrame({"num": [1.5, np.nan]})
+ tooltip_df = pd.DataFrame({"num": [1.54321, np.nan]})
+ data, tooltip_data = get_datatable_data_and_tooltips(data_df, tooltip_df)
+
+ assert data[0] == {"num": 1.5}
+ assert data[1] == {"num": "missing"}
+ assert tooltip_data[0] == {"num": {"value": "1.54321", "type": "text"}}
+ assert tooltip_data[1] == {"num": {"value": "missing", "type": "text"}}
+
def test_round_to_k_1(self):
x = 123456789
expected_r_x = 123000000
diff --git a/tutorial/common/tuto-common01-groups_of_features.ipynb b/tutorial/common/tuto-common01-groups_of_features.ipynb
index 56d34e1f..6b623496 100644
--- a/tutorial/common/tuto-common01-groups_of_features.ipynb
+++ b/tutorial/common/tuto-common01-groups_of_features.ipynb
@@ -45,6 +45,7 @@
"outputs": [],
"source": [
"import pandas as pd\n",
+ "from shapash.utils.dtypes import text_like_columns\n",
"from category_encoders import OrdinalEncoder\n",
"from lightgbm import LGBMRegressor\n",
"from sklearn.model_selection import train_test_split"
@@ -349,11 +350,11 @@
"metadata": {},
"outputs": [],
"source": [
- "categorical_features = [col for col in X.columns if X[col].dtype == 'object']\n",
+ "categorical_features = text_like_columns(X, strict_object=False)\n",
"\n",
"encoder = OrdinalEncoder(\n",
" cols=categorical_features,\n",
- " handle_unknown='ignore',\n",
+ " handle_unknown='return_nan',\n",
" return_df=True\n",
").fit(X)\n",
"\n",
diff --git a/tutorial/common/tuto-common02-colors.ipynb b/tutorial/common/tuto-common02-colors.ipynb
index 18bcfcf1..f5e22438 100644
--- a/tutorial/common/tuto-common02-colors.ipynb
+++ b/tutorial/common/tuto-common02-colors.ipynb
@@ -27,6 +27,7 @@
"outputs": [],
"source": [
"import pandas as pd\n",
+ "from shapash.utils.dtypes import text_like_columns\n",
"from category_encoders import OrdinalEncoder\n",
"from lightgbm import LGBMRegressor\n",
"from sklearn.model_selection import train_test_split"
@@ -324,11 +325,11 @@
"source": [
"from category_encoders import OrdinalEncoder\n",
"\n",
- "categorical_features = [col for col in X_df.columns if X_df[col].dtype == 'object']\n",
+ "categorical_features = text_like_columns(X_df, strict_object=False)\n",
"\n",
"encoder = OrdinalEncoder(\n",
" cols=categorical_features,\n",
- " handle_unknown='ignore',\n",
+ " handle_unknown='return_nan',\n",
" return_df=True).fit(X_df)\n",
"\n",
"X_df=encoder.transform(X_df)"
diff --git a/tutorial/debug_and_what_if/tuto-debug02-recourse-what-if-simulation.ipynb b/tutorial/debug_and_what_if/tuto-debug02-recourse-what-if-simulation.ipynb
index 286de1bb..bccca7a2 100644
--- a/tutorial/debug_and_what_if/tuto-debug02-recourse-what-if-simulation.ipynb
+++ b/tutorial/debug_and_what_if/tuto-debug02-recourse-what-if-simulation.ipynb
@@ -19,6 +19,7 @@
"source": [
"import numpy as np\n",
"import pandas as pd\n",
+ "from shapash.utils.dtypes import text_like_columns\n",
"\n",
"from category_encoders import one_hot\n",
"from sklearn.ensemble import RandomForestRegressor\n",
@@ -137,7 +138,7 @@
"X_raw = house_df.drop(columns=[target_name]).copy()\n",
"y = house_df[[target_name]]\n",
"\n",
- "categorical_cols = X_raw.select_dtypes(include=[\"object\", \"category\"]).columns.tolist()\n",
+ "categorical_cols = text_like_columns(X_raw, strict_object=False)\n",
"numeric_cols = [c for c in X_raw.columns if c not in categorical_cols]\n",
"\n",
"X_train_raw, X_test_raw, y_train, y_test = train_test_split(\n",
diff --git a/tutorial/domain_examples/tuto-domain02-glm-regression.ipynb b/tutorial/domain_examples/tuto-domain02-glm-regression.ipynb
index 74d40cdb..d3f89ebe 100644
--- a/tutorial/domain_examples/tuto-domain02-glm-regression.ipynb
+++ b/tutorial/domain_examples/tuto-domain02-glm-regression.ipynb
@@ -23,6 +23,7 @@
"source": [
"import numpy as np\n",
"import pandas as pd\n",
+ "from shapash.utils.dtypes import text_like_columns\n",
"\n",
"from category_encoders import one_hot\n",
"from sklearn.linear_model import GammaRegressor\n",
@@ -1079,7 +1080,7 @@
"y = house_df['SalePrice']\n",
"X = house_df[house_df.columns.difference(['SalePrice'])].copy()\n",
"\n",
- "categorical_features = [col for col in X.columns if X[col].dtype == 'object']\n",
+ "categorical_features = text_like_columns(X, strict_object=False)\n",
"\n",
"# For GLM, one-hot encoding is safer for nominal categorical variables\n",
"encoder = one_hot.OneHotEncoder(\n",
diff --git a/tutorial/explainability_quality/tuto-quality01-Builing-confidence-explainability.ipynb b/tutorial/explainability_quality/tuto-quality01-Builing-confidence-explainability.ipynb
index 1dad7fae..20e6bee8 100644
--- a/tutorial/explainability_quality/tuto-quality01-Builing-confidence-explainability.ipynb
+++ b/tutorial/explainability_quality/tuto-quality01-Builing-confidence-explainability.ipynb
@@ -28,6 +28,7 @@
"outputs": [],
"source": [
"import pandas as pd\n",
+ "from shapash.utils.dtypes import text_like_columns\n",
"from sklearn.ensemble import ExtraTreesClassifier\n",
"from sklearn.model_selection import train_test_split"
]
@@ -217,11 +218,11 @@
"source": [
"from category_encoders import OrdinalEncoder\n",
"\n",
- "categorical_features = [col for col in X_df.columns if X_df[col].dtype == 'object']\n",
+ "categorical_features = text_like_columns(X_df, strict_object=False)\n",
"\n",
"encoder = OrdinalEncoder(\n",
" cols=categorical_features,\n",
- " handle_unknown='ignore',\n",
+ " handle_unknown='return_nan',\n",
" return_df=True).fit(X_df)\n",
"\n",
"X_df=encoder.transform(X_df)"
diff --git a/tutorial/explainer_and_backend/tuto-expl01-Shapash-Viz-using-Shap-contributions.ipynb b/tutorial/explainer_and_backend/tuto-expl01-Shapash-Viz-using-Shap-contributions.ipynb
index 6aa19cc6..eefb4327 100644
--- a/tutorial/explainer_and_backend/tuto-expl01-Shapash-Viz-using-Shap-contributions.ipynb
+++ b/tutorial/explainer_and_backend/tuto-expl01-Shapash-Viz-using-Shap-contributions.ipynb
@@ -26,10 +26,13 @@
},
{
"cell_type": "code",
- "execution_count": 1,
+ "execution_count": null,
"metadata": {},
"outputs": [],
"source": [
+ "import warnings\n",
+ "warnings.filterwarnings(\"ignore\")\n",
+ "\n",
"import pandas as pd\n",
"from category_encoders import OrdinalEncoder\n",
"from sklearn.ensemble import RandomForestClassifier\n",
@@ -230,7 +233,7 @@
"outputs": [],
"source": [
"categ_encoding = OrdinalEncoder(cols=varcat, \\\n",
- " handle_unknown='ignore', \\\n",
+ " handle_unknown='return_nan', \\\n",
" return_df=True).fit(X)\n",
"X = categ_encoding.transform(X)"
]
diff --git a/tutorial/explainer_and_backend/tuto-expl02-Shapash-Viz-using-Lime-contributions.ipynb b/tutorial/explainer_and_backend/tuto-expl02-Shapash-Viz-using-Lime-contributions.ipynb
index 065de7a2..acdee1c7 100644
--- a/tutorial/explainer_and_backend/tuto-expl02-Shapash-Viz-using-Lime-contributions.ipynb
+++ b/tutorial/explainer_and_backend/tuto-expl02-Shapash-Viz-using-Lime-contributions.ipynb
@@ -225,7 +225,7 @@
"outputs": [],
"source": [
"categ_encoding = OrdinalEncoder(cols=varcat, \\\n",
- " handle_unknown='ignore', \\\n",
+ " handle_unknown='return_nan', \\\n",
" return_df=True).fit(X)\n",
"X = categ_encoding.transform(X)"
]
diff --git a/tutorial/explainer_and_backend/tuto-expl04-Shapash-compute-Lime-faster.ipynb b/tutorial/explainer_and_backend/tuto-expl04-Shapash-compute-Lime-faster.ipynb
index b71ed202..badc7df5 100644
--- a/tutorial/explainer_and_backend/tuto-expl04-Shapash-compute-Lime-faster.ipynb
+++ b/tutorial/explainer_and_backend/tuto-expl04-Shapash-compute-Lime-faster.ipynb
@@ -43,6 +43,7 @@
"source": [
"import numpy as np\n",
"import pandas as pd\n",
+ "from shapash.utils.dtypes import text_like_columns\n",
"from category_encoders import OrdinalEncoder\n",
"from sklearn.ensemble import RandomForestClassifier\n",
"from sklearn.model_selection import train_test_split\n",
@@ -136,11 +137,11 @@
"metadata": {},
"outputs": [],
"source": [
- "categorical_features = [col for col in X_df.columns if X_df[col].dtype == 'object']\n",
+ "categorical_features = text_like_columns(X_df, strict_object=False)\n",
"\n",
"encoder = OrdinalEncoder(\n",
" cols=categorical_features,\n",
- " handle_unknown='ignore',\n",
+ " handle_unknown='return_nan',\n",
" return_df=True).fit(X_df)\n",
"\n",
"X_df=encoder.transform(X_df)"
diff --git a/tutorial/explainer_and_backend/tuto-expl05-Shapash-using-Fasttreeshap.ipynb b/tutorial/explainer_and_backend/tuto-expl05-Shapash-using-Fasttreeshap.ipynb
index baa86e53..a25a1d02 100644
--- a/tutorial/explainer_and_backend/tuto-expl05-Shapash-using-Fasttreeshap.ipynb
+++ b/tutorial/explainer_and_backend/tuto-expl05-Shapash-using-Fasttreeshap.ipynb
@@ -31,6 +31,7 @@
"source": [
"import numpy as np\n",
"import pandas as pd\n",
+ "from shapash.utils.dtypes import text_like_columns\n",
"from category_encoders import OrdinalEncoder\n",
"from lightgbm import LGBMRegressor\n",
"from sklearn.model_selection import train_test_split\n",
@@ -82,11 +83,11 @@
"source": [
"from category_encoders import OrdinalEncoder\n",
"\n",
- "categorical_features = [col for col in X_df.columns if X_df[col].dtype == 'object']\n",
+ "categorical_features = text_like_columns(X_df, strict_object=False)\n",
"\n",
"encoder = OrdinalEncoder(\n",
" cols=categorical_features,\n",
- " handle_unknown='ignore',\n",
+ " handle_unknown='return_nan',\n",
" return_df=True).fit(X_df)\n",
"\n",
"X_df=encoder.transform(X_df)"
diff --git a/tutorial/explainer_and_backend/tuto-expl06-Shapash-custom-backend.ipynb b/tutorial/explainer_and_backend/tuto-expl06-Shapash-custom-backend.ipynb
index d3cbc22f..f454e677 100644
--- a/tutorial/explainer_and_backend/tuto-expl06-Shapash-custom-backend.ipynb
+++ b/tutorial/explainer_and_backend/tuto-expl06-Shapash-custom-backend.ipynb
@@ -26,6 +26,7 @@
"outputs": [],
"source": [
"import pandas as pd\n",
+ "from shapash.utils.dtypes import text_like_columns\n",
"\n",
"from sklearn.ensemble import RandomForestClassifier, RandomForestRegressor\n",
"from category_encoders import OrdinalEncoder, OneHotEncoder, TargetEncoder\n",
@@ -432,11 +433,11 @@
"source": [
"# Encode categorical features \n",
"\n",
- "categorical_features = [col for col in X2.columns if X2[col].dtype == 'object']\n",
+ "categorical_features = text_like_columns(X2, strict_object=False)\n",
"\n",
"encoder2 = OrdinalEncoder(\n",
" cols=categorical_features, \n",
- " handle_unknown='ignore', \n",
+ " handle_unknown='return_nan', \n",
" return_df=True\n",
").fit(X2)\n",
"\n",
diff --git a/tutorial/explainer_and_backend/tuto-expl07-Shapash-using-ShapIQ.ipynb b/tutorial/explainer_and_backend/tuto-expl07-Shapash-using-ShapIQ.ipynb
index fd765332..46988450 100644
--- a/tutorial/explainer_and_backend/tuto-expl07-Shapash-using-ShapIQ.ipynb
+++ b/tutorial/explainer_and_backend/tuto-expl07-Shapash-using-ShapIQ.ipynb
@@ -39,6 +39,7 @@
"import warnings\n",
"import numpy as np\n",
"import pandas as pd\n",
+ "from shapash.utils.dtypes import text_like_columns\n",
"from category_encoders import OrdinalEncoder\n",
"from lightgbm import LGBMRegressor\n",
"from sklearn.model_selection import train_test_split\n",
@@ -96,11 +97,11 @@
"metadata": {},
"outputs": [],
"source": [
- "categorical_features = [col for col in X_df.columns if X_df[col].dtype == 'object']\n",
+ "categorical_features = text_like_columns(X_df, strict_object=False)\n",
"\n",
"encoder = OrdinalEncoder(\n",
" cols=categorical_features,\n",
- " handle_unknown='ignore',\n",
+ " handle_unknown='return_nan',\n",
" return_df=True\n",
").fit(X_df)\n",
"\n",
diff --git a/tutorial/generate_report/shapash_report_example.py b/tutorial/generate_report/shapash_report_example.py
index 4734a587..79fe6cf9 100644
--- a/tutorial/generate_report/shapash_report_example.py
+++ b/tutorial/generate_report/shapash_report_example.py
@@ -15,15 +15,16 @@
from shapash import SmartExplainer
from shapash.data.data_loader import data_loading
+from shapash.utils.dtypes import text_like_columns
if __name__ == "__main__":
house_df, house_dict = data_loading("house_prices")
y_df = house_df["SalePrice"]
X_df = house_df[house_df.columns.difference(["SalePrice"])]
- categorical_features = [col for col in X_df.columns if X_df[col].dtype == "object"]
+ categorical_features = text_like_columns(X_df, strict_object=False)
- encoder = OrdinalEncoder(cols=categorical_features, handle_unknown="ignore", return_df=True).fit(X_df)
+ encoder = OrdinalEncoder(cols=categorical_features, handle_unknown="return_nan", return_df=True).fit(X_df)
X_df = encoder.transform(X_df)
diff --git a/tutorial/generate_report/tuto-shapash-report01.ipynb b/tutorial/generate_report/tuto-shapash-report01.ipynb
index 53ae5906..6b85bff6 100644
--- a/tutorial/generate_report/tuto-shapash-report01.ipynb
+++ b/tutorial/generate_report/tuto-shapash-report01.ipynb
@@ -45,6 +45,7 @@
"outputs": [],
"source": [
"import pandas as pd\n",
+ "from shapash.utils.dtypes import text_like_columns\n",
"from category_encoders import OrdinalEncoder\n",
"from sklearn.ensemble import RandomForestRegressor\n",
"from sklearn.model_selection import train_test_split"
@@ -80,11 +81,11 @@
"source": [
"from category_encoders import OrdinalEncoder\n",
"\n",
- "categorical_features = [col for col in X_df.columns if X_df[col].dtype == 'object']\n",
+ "categorical_features = text_like_columns(X_df, strict_object=False)\n",
"\n",
"encoder = OrdinalEncoder(\n",
" cols=categorical_features,\n",
- " handle_unknown='ignore',\n",
+ " handle_unknown='return_nan',\n",
" return_df=True).fit(X_df)\n",
"\n",
"X_df = encoder.transform(X_df)"
diff --git a/tutorial/plots_and_charts/tuto-plot01-local_plot-and-to_pandas.ipynb b/tutorial/plots_and_charts/tuto-plot01-local_plot-and-to_pandas.ipynb
index c343cdec..11ec2fc9 100644
--- a/tutorial/plots_and_charts/tuto-plot01-local_plot-and-to_pandas.ipynb
+++ b/tutorial/plots_and_charts/tuto-plot01-local_plot-and-to_pandas.ipynb
@@ -30,6 +30,7 @@
"outputs": [],
"source": [
"import pandas as pd\n",
+ "from shapash.utils.dtypes import text_like_columns\n",
"from category_encoders import OrdinalEncoder\n",
"from catboost import CatBoostRegressor, CatBoostClassifier\n",
"from sklearn.model_selection import train_test_split"
@@ -62,11 +63,11 @@
"source": [
"from category_encoders import OrdinalEncoder\n",
"\n",
- "categorical_features = [col for col in X_df.columns if X_df[col].dtype == 'object']\n",
+ "categorical_features = text_like_columns(X_df, strict_object=False)\n",
"\n",
"encoder = OrdinalEncoder(\n",
" cols=categorical_features,\n",
- " handle_unknown='ignore',\n",
+ " handle_unknown='return_nan',\n",
" return_df=True).fit(X_df)\n",
"\n",
"X_df=encoder.transform(X_df)"
diff --git a/tutorial/plots_and_charts/tuto-plot02-contribution_plot.ipynb b/tutorial/plots_and_charts/tuto-plot02-contribution_plot.ipynb
index 6810d3ad..0bbb8aee 100644
--- a/tutorial/plots_and_charts/tuto-plot02-contribution_plot.ipynb
+++ b/tutorial/plots_and_charts/tuto-plot02-contribution_plot.ipynb
@@ -29,6 +29,7 @@
"outputs": [],
"source": [
"import pandas as pd\n",
+ "from shapash.utils.dtypes import text_like_columns\n",
"from xgboost import XGBClassifier\n",
"from sklearn.model_selection import train_test_split"
]
@@ -230,11 +231,11 @@
"source": [
"from category_encoders import OrdinalEncoder\n",
"\n",
- "categorical_features = [col for col in X_df.columns if X_df[col].dtype == 'object']\n",
+ "categorical_features = text_like_columns(X_df, strict_object=False)\n",
"\n",
"encoder = OrdinalEncoder(\n",
" cols=categorical_features,\n",
- " handle_unknown='ignore',\n",
+ " handle_unknown='return_nan',\n",
" return_df=True).fit(X_df)\n",
"\n",
"X_df=encoder.transform(X_df)"
diff --git a/tutorial/plots_and_charts/tuto-plot03-features-importance.ipynb b/tutorial/plots_and_charts/tuto-plot03-features-importance.ipynb
index e3279892..b647bf5d 100644
--- a/tutorial/plots_and_charts/tuto-plot03-features-importance.ipynb
+++ b/tutorial/plots_and_charts/tuto-plot03-features-importance.ipynb
@@ -28,6 +28,7 @@
"metadata": {},
"outputs": [],
"source": [
+ "from shapash.utils.dtypes import text_like_columns\n",
"from sklearn.ensemble import ExtraTreesClassifier\n",
"from sklearn.model_selection import train_test_split"
]
@@ -215,11 +216,11 @@
"source": [
"from category_encoders import OrdinalEncoder\n",
"\n",
- "categorical_features = [col for col in X_df.columns if X_df[col].dtype == 'object']\n",
+ "categorical_features = text_like_columns(X_df, strict_object=False)\n",
"\n",
"encoder = OrdinalEncoder(\n",
" cols=categorical_features,\n",
- " handle_unknown='ignore',\n",
+ " handle_unknown='return_nan',\n",
" return_df=True).fit(X_df)\n",
"\n",
"X_df=encoder.transform(X_df)"
diff --git a/tutorial/plots_and_charts/tuto-plot04-compare_plot.ipynb b/tutorial/plots_and_charts/tuto-plot04-compare_plot.ipynb
index bce100b2..3c1c8274 100644
--- a/tutorial/plots_and_charts/tuto-plot04-compare_plot.ipynb
+++ b/tutorial/plots_and_charts/tuto-plot04-compare_plot.ipynb
@@ -31,6 +31,7 @@
"outputs": [],
"source": [
"import pandas as pd\n",
+ "from shapash.utils.dtypes import text_like_columns\n",
"from catboost import CatBoostRegressor\n",
"from sklearn.model_selection import train_test_split"
]
@@ -328,11 +329,11 @@
"source": [
"from category_encoders import OrdinalEncoder\n",
"\n",
- "categorical_features = [col for col in X_df.columns if X_df[col].dtype == 'object']\n",
+ "categorical_features = text_like_columns(X_df, strict_object=False)\n",
"\n",
"encoder = OrdinalEncoder(\n",
" cols=categorical_features,\n",
- " handle_unknown='ignore',\n",
+ " handle_unknown='return_nan',\n",
" return_df=True).fit(X_df)\n",
"\n",
"X_df = encoder.transform(X_df)"
diff --git a/tutorial/plots_and_charts/tuto-plot05-interactions-plot.ipynb b/tutorial/plots_and_charts/tuto-plot05-interactions-plot.ipynb
index 25eee5e1..c5d1feab 100644
--- a/tutorial/plots_and_charts/tuto-plot05-interactions-plot.ipynb
+++ b/tutorial/plots_and_charts/tuto-plot05-interactions-plot.ipynb
@@ -32,6 +32,7 @@
"outputs": [],
"source": [
"import pandas as pd\n",
+ "from shapash.utils.dtypes import text_like_columns\n",
"from category_encoders import OrdinalEncoder\n",
"from xgboost import XGBClassifier\n",
"from sklearn.model_selection import train_test_split"
@@ -211,11 +212,11 @@
"source": [
"from category_encoders import OrdinalEncoder\n",
"\n",
- "categorical_features = [col for col in X_df.columns if X_df[col].dtype == 'object']\n",
+ "categorical_features = text_like_columns(X_df, strict_object=False)\n",
"\n",
"encoder = OrdinalEncoder(\n",
" cols=categorical_features,\n",
- " handle_unknown='ignore',\n",
+ " handle_unknown='return_nan',\n",
" return_df=True).fit(X_df)\n",
"\n",
"X_df=encoder.transform(X_df)"
diff --git a/tutorial/plots_and_charts/tuto-plot06-prediction_plot.ipynb b/tutorial/plots_and_charts/tuto-plot06-prediction_plot.ipynb
index 68e206cb..cf6581fb 100644
--- a/tutorial/plots_and_charts/tuto-plot06-prediction_plot.ipynb
+++ b/tutorial/plots_and_charts/tuto-plot06-prediction_plot.ipynb
@@ -35,7 +35,8 @@
"source": [
"from xgboost import XGBClassifier\n",
"from catboost import CatBoostRegressor\n",
- "from sklearn.model_selection import train_test_split"
+ "from sklearn.model_selection import train_test_split\n",
+ "from shapash.utils.dtypes import text_like_columns"
]
},
{
@@ -65,11 +66,11 @@
"source": [
"from category_encoders import OrdinalEncoder\n",
"\n",
- "categorical_features = [col for col in X_df.columns if X_df[col].dtype == 'object']\n",
+ "categorical_features = text_like_columns(X_df, strict_object=False)\n",
"\n",
"encoder = OrdinalEncoder(\n",
" cols=categorical_features,\n",
- " handle_unknown='ignore',\n",
+ " handle_unknown='return_nan',\n",
" return_df=True).fit(X_df)\n",
"\n",
"X_df=encoder.transform(X_df)\n",
@@ -178,11 +179,11 @@
"source": [
"from category_encoders import OrdinalEncoder\n",
"\n",
- "categorical_features = [col for col in X_df.columns if X_df[col].dtype == 'object']\n",
+ "categorical_features = text_like_columns(X_df, strict_object=False)\n",
"\n",
"encoder = OrdinalEncoder(\n",
" cols=categorical_features,\n",
- " handle_unknown='ignore',\n",
+ " handle_unknown='return_nan',\n",
" return_df=True).fit(X_df)\n",
"\n",
"X_df=encoder.transform(X_df)\n",
diff --git a/tutorial/plots_and_charts/tuto-plot07-additional_plots_visualizations.ipynb b/tutorial/plots_and_charts/tuto-plot07-additional_plots_visualizations.ipynb
index bdfe811f..46e53290 100644
--- a/tutorial/plots_and_charts/tuto-plot07-additional_plots_visualizations.ipynb
+++ b/tutorial/plots_and_charts/tuto-plot07-additional_plots_visualizations.ipynb
@@ -24,7 +24,8 @@
"from shapash import SmartExplainer\n",
"from shapash.plots.plot_correlations import plot_correlations\n",
"from shapash.plots.plot_univariate import plot_distribution\n",
- "from shapash.plots.plot_evaluation_metrics import plot_confusion_matrix"
+ "from shapash.plots.plot_evaluation_metrics import plot_confusion_matrix\n",
+ "from shapash.utils.dtypes import text_like_columns"
]
},
{
@@ -282,11 +283,11 @@
"outputs": [],
"source": [
"# Identify categorical features\n",
- "categorical_features = [col for col in X_df.columns if X_df[col].dtype == 'object']\n",
+ "categorical_features = text_like_columns(X_df, strict_object=False)\n",
"\n",
"# Apply encoding\n",
- "encoder = OrdinalEncoder(cols=categorical_features, handle_unknown='ignore').fit(X_df)\n",
- "encoder_target = OrdinalEncoder(cols=['Pclass'], handle_unknown='ignore').fit(y_df)\n",
+ "encoder = OrdinalEncoder(cols=categorical_features, handle_unknown='return_nan').fit(X_df)\n",
+ "encoder_target = OrdinalEncoder(cols=['Pclass'], handle_unknown='return_nan').fit(y_df)\n",
"\n",
"X_df = encoder.transform(X_df)\n",
"y_df = encoder_target.transform(y_df)"
diff --git a/tutorial/postprocess/tuto-postprocess01.ipynb b/tutorial/postprocess/tuto-postprocess01.ipynb
index 69fe72c3..5093ebcc 100644
--- a/tutorial/postprocess/tuto-postprocess01.ipynb
+++ b/tutorial/postprocess/tuto-postprocess01.ipynb
@@ -28,6 +28,7 @@
"outputs": [],
"source": [
"import pandas as pd\n",
+ "from shapash.utils.dtypes import text_like_columns\n",
"from sklearn.model_selection import train_test_split\n",
"from sklearn.ensemble import RandomForestClassifier"
]
@@ -229,11 +230,11 @@
"source": [
"from category_encoders import OrdinalEncoder\n",
"\n",
- "categorical_features = [col for col in X_df.columns if X_df[col].dtype == 'object']\n",
+ "categorical_features = text_like_columns(X_df, strict_object=False)\n",
"\n",
"encoder = OrdinalEncoder(\n",
" cols=categorical_features,\n",
- " handle_unknown='ignore',\n",
+ " handle_unknown='return_nan',\n",
" return_df=True).fit(X_df)\n",
"\n",
"X_df = encoder.transform(X_df)"
diff --git a/tutorial/predictor_to_production/tuto-smartpredictor-introduction-to-SmartPredictor.ipynb b/tutorial/predictor_to_production/tuto-smartpredictor-introduction-to-SmartPredictor.ipynb
index 74265363..46caa6ad 100644
--- a/tutorial/predictor_to_production/tuto-smartpredictor-introduction-to-SmartPredictor.ipynb
+++ b/tutorial/predictor_to_production/tuto-smartpredictor-introduction-to-SmartPredictor.ipynb
@@ -275,7 +275,7 @@
"outputs": [],
"source": [
"categ_encoding = OrdinalEncoder(cols=varcat, \\\n",
- " handle_unknown='ignore', \\\n",
+ " handle_unknown='return_nan', \\\n",
" return_df=True).fit(X)\n",
"X = categ_encoding.transform(X)"
]
diff --git a/tutorial/production_and_ops/tuto-prod01-overview-model-in-production.ipynb b/tutorial/production_and_ops/tuto-prod01-overview-model-in-production.ipynb
index 9b9f962e..22c30120 100644
--- a/tutorial/production_and_ops/tuto-prod01-overview-model-in-production.ipynb
+++ b/tutorial/production_and_ops/tuto-prod01-overview-model-in-production.ipynb
@@ -30,6 +30,7 @@
"outputs": [],
"source": [
"import pandas as pd\n",
+ "from shapash.utils.dtypes import text_like_columns\n",
"from category_encoders import OrdinalEncoder\n",
"from lightgbm import LGBMRegressor\n",
"from sklearn.model_selection import train_test_split"
@@ -98,10 +99,10 @@
"source": [
"from category_encoders import OrdinalEncoder\n",
"\n",
- "categorical_features = [col for col in X_df.columns if X_df[col].dtype == 'object']\n",
+ "categorical_features = text_like_columns(X_df, strict_object=False)\n",
"\n",
"encoder = OrdinalEncoder(cols=categorical_features,\n",
- " handle_unknown='ignore',\n",
+ " handle_unknown='return_nan',\n",
" return_df=True).fit(X_df)\n",
"\n",
"X_encoded=encoder.transform(X_df)"
@@ -2177,7 +2178,7 @@
"celltoolbar": "Aucun(e)",
"hide_input": false,
"kernelspec": {
- "display_name": "survptf_312",
+ "display_name": "Python 3",
"language": "python",
"name": "python3"
},
diff --git a/tutorial/production_and_ops/tuto-prod03-batch-scoring-parquet.ipynb b/tutorial/production_and_ops/tuto-prod03-batch-scoring-parquet.ipynb
index 175082ea..ab6ee705 100644
--- a/tutorial/production_and_ops/tuto-prod03-batch-scoring-parquet.ipynb
+++ b/tutorial/production_and_ops/tuto-prod03-batch-scoring-parquet.ipynb
@@ -470,9 +470,9 @@
" scored_chunk[\"chunk_id\"] = chunk_id\n",
"\n",
" # Parquet cannot serialize mixed python object columns reliably (e.g. int/str in value_i).\n",
- " object_cols = scored_chunk.select_dtypes(include=[\"object\"]).columns\n",
- " if len(object_cols) > 0:\n",
- " scored_chunk[object_cols] = scored_chunk[object_cols].astype(\"string\")\n",
+ " string_cols = scored_chunk.select_dtypes(include=[\"object\", \"string\"]).columns\n",
+ " if len(string_cols) > 0:\n",
+ " scored_chunk[string_cols] = scored_chunk[string_cols].astype(\"string\")\n",
"\n",
" table = pa.Table.from_pandas(scored_chunk, preserve_index=False)\n",
" if writer is None:\n",
diff --git a/tutorial/tutorial01-Shapash-Overview-Launch-WebApp.ipynb b/tutorial/tutorial01-Shapash-Overview-Launch-WebApp.ipynb
index 2a99345d..eedee778 100644
--- a/tutorial/tutorial01-Shapash-Overview-Launch-WebApp.ipynb
+++ b/tutorial/tutorial01-Shapash-Overview-Launch-WebApp.ipynb
@@ -27,6 +27,7 @@
"outputs": [],
"source": [
"import pandas as pd\n",
+ "from shapash.utils.dtypes import text_like_columns\n",
"from category_encoders import OrdinalEncoder\n",
"from lightgbm import LGBMRegressor\n",
"from sklearn.model_selection import train_test_split\n",
@@ -347,11 +348,11 @@
"source": [
"from category_encoders import OrdinalEncoder\n",
"\n",
- "categorical_features = [col for col in X_df.columns if X_df[col].dtype == 'object']\n",
+ "categorical_features = text_like_columns(X_df, strict_object=False)\n",
"\n",
"encoder = OrdinalEncoder(\n",
" cols=categorical_features,\n",
- " handle_unknown='ignore',\n",
+ " handle_unknown='return_nan',\n",
" return_df=True).fit(X_df)\n",
"\n",
"X_df=encoder.transform(X_df)"
diff --git a/tutorial/tutorial02-Shapash-overview-in-Jupyter.ipynb b/tutorial/tutorial02-Shapash-overview-in-Jupyter.ipynb
index c03b1780..b1eae105 100644
--- a/tutorial/tutorial02-Shapash-overview-in-Jupyter.ipynb
+++ b/tutorial/tutorial02-Shapash-overview-in-Jupyter.ipynb
@@ -27,6 +27,7 @@
"outputs": [],
"source": [
"import pandas as pd\n",
+ "from shapash.utils.dtypes import text_like_columns\n",
"from category_encoders import OrdinalEncoder\n",
"from lightgbm import LGBMRegressor\n",
"from sklearn.model_selection import train_test_split"
@@ -326,11 +327,11 @@
"source": [
"from category_encoders import OrdinalEncoder\n",
"\n",
- "categorical_features = [col for col in X_df.columns if X_df[col].dtype == 'object']\n",
+ "categorical_features = text_like_columns(X_df, strict_object=False)\n",
"\n",
"encoder = OrdinalEncoder(\n",
" cols=categorical_features,\n",
- " handle_unknown='ignore',\n",
+ " handle_unknown='return_nan',\n",
" return_df=True).fit(X_df)\n",
"\n",
"X_df=encoder.transform(X_df)"