Unify missing values display

.github/workflows/main.yml

@@ -34,6 +34,8 @@ jobs:
       run: make lint
     - name: Check formatting
       run: make format
+    - name: Check type annotations
+      run: make typecheck
 
   test:
     name: Build & Test
@@ -64,3 +66,31 @@ jobs:
         python -m build
     - name: Test and coverage
       run: make coverage
+
+  pandas_compat:
+    name: Pandas Compat (${{ matrix.pandas }})
+    needs: code_quality
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+    strategy:
+      fail-fast: false
+      matrix:
+        pandas: ["2.2.2", "2.3.*", "3.*"]
+    steps:
+    - uses: actions/checkout@v6
+      with:
+        fetch-depth: 0  # required for setuptools-scm to read git tags
+    - name: Set up Python 3.12
+      uses: actions/setup-python@v6
+      with:
+        python-version: "3.12"
+        cache: pip
+        cache-dependency-path: pyproject.toml
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        python -m pip install '.[all]'
+        python -m pip install "pandas==${{ matrix.pandas }}"
+        python -c "import pandas; print(pandas.__version__)"
+    - name: Test and coverage
+      run: make coverage

.pre-commit-config.yaml

@@ -36,6 +36,18 @@ repos:
         args: [--fix]
       # Run the formatter
       - id: ruff-format
+-   repo: local
+    hooks:
+    -   id: mypy
+        name: mypy
+        # Runs from the local dev venv (not an isolated pre-commit env) so it sees
+        # shapash's actual dependencies (dash, pandas, sklearn, ...); pyproject.toml
+        # sets ignore_missing_imports=true, which would silently mask them as Any
+        # otherwise. Requires `.venv` to be set up.
+        entry: mypy shapash
+        language: system
+        pass_filenames: false
+        files: ^shapash/.*\.py$
 -   repo: https://github.com/adamchainz/blacken-docs
     rev: 1.20.0
     hooks:

CONTRIBUTING.md

@@ -113,6 +113,11 @@ ruff check
 ruff format
 ```
 
+Check your type annotations with mypy:
+```
+mypy shapash
+```
+
 ## Commit your changes
 
 We recommend committing with clear messages and grouping your commits by modifications dependencies.

Makefile

@@ -56,6 +56,9 @@ lint: ## check style with ruff
 format: ## check formatting with ruff
 	ruff format --check
 
+typecheck: ## check type annotations with mypy
+	mypy shapash
+
 test: ## run tests quickly with the default Python
 	pytest
 

pyproject.toml

@@ -39,8 +39,9 @@ dependencies = [
     "matplotlib>=3.8.4",
     "nbformat>5.8.0",
     "numba>=0.60.0",
-    "numpy>=2.0.0,<2.6.0",
-    "pandas>=2.2.2,<3.0.4",
+    "numpy>=2.0.0",
+    # 3.0.4 yanked upstream: segfaults in datetime ops (pandas #66083)
+    "pandas>=2.2.2,!=3.0.4,<4.0.0",
     "plotly>=5.0.0,<6.0.0",
     "scikit-learn>=1.8.0,<1.9.0",
     "scipy>=1.13.0",

shapash/backend/base_backend.py

@@ -41,7 +41,7 @@ def __init__(self, model: Any, preprocessing: Any | None = None):
         self.model = model
         self.preprocessing = preprocessing
         self.explain_data: Any = None
-        self.state = None
+        self.state: Any = None
         self._case, self._classes = check_model(model)
         if self._case not in self.supported_cases:
             raise ValueError(f"Model not supported by the backend as it does not cover {self._case} case")

shapash/backend/lime_backend.py

@@ -1,3 +1,5 @@
+from collections.abc import Callable
+
 try:
     from lime import lime_tabular
 
@@ -115,7 +117,7 @@ def _explain_multiclass(
         self,
         x: pd.DataFrame,
         feature_names: list,
-        predict_fn: callable,
+        predict_fn: Callable,
         num_classes: int,
     ) -> list[pd.DataFrame]:
         """
@@ -152,7 +154,7 @@ def _explain_binary_or_regression(
         self,
         x: pd.DataFrame,
         feature_names: list,
-        predict_fn: callable,
+        predict_fn: Callable,
     ) -> pd.DataFrame:
         """
         Compute LIME contributions for binary classification or regression.

shapash/explainer/consistency.py

@@ -10,6 +10,7 @@
 from sklearn.manifold import MDS
 
 from shapash.style.style_utils import colors_loading, define_style, select_palette
+from shapash.utils.dtypes import text_like_columns
 from shapash.utils.utils import adjust_title_height
 
 
@@ -576,8 +577,8 @@ def plot_pairwise_consistency(
         if isinstance(self.preprocessing, OrdinalEncoder):
             encoder = self.preprocessing
         else:
-            categorical_features = [col for col in x.columns if x[col].dtype == "object"]
-            encoder = OrdinalEncoder(cols=categorical_features, handle_unknown="ignore", return_df=True).fit(x)
+            categorical_features = text_like_columns(x, strict_object=False)
+            encoder = OrdinalEncoder(cols=categorical_features, handle_unknown="return_nan", return_df=True).fit(x)
             x = encoder.transform(x)
 
         xaxis_title = (

shapash/explainer/smart_explainer.py

@@ -6,6 +6,7 @@
 import logging
 import shutil
 import tempfile
+from typing import Any
 
 import numpy as np
 import pandas as pd
@@ -195,7 +196,7 @@ def __init__(
         features_groups=None,
         features_dict=None,
         label_dict=None,
-        title_story: str = None,
+        title_story: str | None = None,
         palette_name=None,
         colors_dict=None,
         **backend_kwargs,
@@ -246,7 +247,7 @@ def __init__(
         self.features_compacity = None
         self.contributions = None
         self.explain_data = None
-        self.features_imp = None
+        self.features_imp: Any = None
 
     def compile(
         self,
@@ -386,7 +387,7 @@ def _compile_features_groups(self, features_groups):
             raise AssertionError(f"Selected backend ({self.backend.name}) does not support groups of features.")
         # Compute contributions for groups of features
         self.contributions_groups = self.state.compute_grouped_contributions(self.contributions, features_groups)
-        self.features_imp_groups = None
+        self.features_imp_groups: Any = None
         # Update features dict with groups names
         self._update_features_dict_with_groups(features_groups=features_groups)
         # Compute t-sne projections for groups of features
@@ -478,7 +479,7 @@ def add(
         y_target=None,
         label_dict=None,
         features_dict=None,
-        title_story: str = None,
+        title_story: str | None = None,
         columns_order=None,
         additional_data=None,
         additional_features_dict=None,
@@ -1400,7 +1401,7 @@ def compute_features_compacity(self, selection, distance, nb_features):
 
         self.features_compacity = {"features_needed": features_needed, "distance_reached": distance_reached}
 
-    def init_app(self, settings: dict = None):
+    def init_app(self, settings: dict | None = None):
         """
         Initialize a SmartApp instance for the current SmartExplainer object.
 
@@ -1436,10 +1437,10 @@ def init_app(self, settings: dict = None):
 
     def run_app(
         self,
-        port: int = None,
-        host: str = None,
-        title_story: str = None,
-        settings: dict = None,
+        port: int | None = None,
+        host: str | None = None,
+        title_story: str | None = None,
+        settings: dict | None = None,
     ) -> CustomThread:
         """
         Launch the Shapash interpretability WebApp associated with this SmartExplainer.
@@ -1502,13 +1503,7 @@ def run_app(
                 port = 8050
             host_name = get_host_name()
             wsgi_server = make_server(host, port, self.smartapp.server)
-            server_instance = CustomThread(target=wsgi_server.serve_forever)
-
-            def _kill():
-                wsgi_server.shutdown()
-                server_instance.killed = True
-
-            server_instance.kill = _kill
+            server_instance = CustomThread(target=wsgi_server.serve_forever, on_kill=wsgi_server.shutdown)
             if host_name is None:
                 host_name = host
             elif host != DEFAULT_HOST:

shapash/explainer/smart_plotter.py

@@ -38,6 +38,7 @@
     adjust_title_height,
     compute_digit_number,
     compute_sorted_variables_interactions_list_indices,
+    format_missing_value,
     maximum_difference_sort_value,
     top_contributors,
     truncate_str,
@@ -2345,7 +2346,7 @@ def clustering_by_explainability_plot(
                     for idx, row in df_pred.iterrows():
                         text = f"Id: {idx}<br />"
                         if el not in ["predictions", "targets", "errors"]:
-                            text += f"{el}: {row[el]}<br />"
+                            text += f"{el}: {format_missing_value(row[el])}<br />"
                         text += f"Predicted Value: {row['proba_values']:.{self._round_digit}f}<br />"
                         if "error" in df_pred.columns:
                             text += f"Error: {row['error']:.{self._round_digit}f}<br />"
@@ -2360,19 +2361,30 @@ def clustering_by_explainability_plot(
                         if el not in ["predictions", "targets", "errors"]:
                             is_num = is_numeric_dtype(df_pred[el]) and not is_bool_dtype(df_pred[el])
                             n_unique = df_pred[el].nunique(dropna=True)
+                            cluster_values = df_pred.loc[df_pred["cluster"] == c, el]
                             if is_num and n_unique > 5:
-                                mean_el = df_pred.loc[df_pred["cluster"] == c, el].mean()
-                                std_el = df_pred.loc[df_pred["cluster"] == c, el].std()
-                                hv_text_cluster += f"<br />{el} mean: {mean_el:.{compute_digit_number(mean_el, 3)}f}"
-                                hv_text_cluster += f"<br />{el} std: {std_el:.{compute_digit_number(std_el, 3)}f}"
+                                mean_el = cluster_values.mean()
+                                std_el = cluster_values.std()
+                                if pd.isna(mean_el) or pd.isna(std_el):
+                                    hv_text_cluster += f"<br />{el} mean: {format_missing_value(mean_el)}"
+                                    hv_text_cluster += f"<br />{el} std: {format_missing_value(std_el)}"
+                                else:
+                                    hv_text_cluster += (
+                                        f"<br />{el} mean: {mean_el:.{compute_digit_number(mean_el, 3)}f}"
+                                    )
+                                    hv_text_cluster += f"<br />{el} std: {std_el:.{compute_digit_number(std_el, 3)}f}"
                             else:
-                                top_element = df_pred.loc[df_pred["cluster"] == c, el].mode()[0]
-                                top_element_percentage = (
-                                    np.sum(df_pred.loc[df_pred["cluster"] == c, el] == top_element)
-                                    / df_pred.loc[df_pred["cluster"] == c, el].size
-                                    * 100
+                                # dropna=False so that null values can be reported as the top modality
+                                top_element = cluster_values.mode(dropna=False).iloc[0]
+                                if pd.isna(top_element):
+                                    top_element_count = cluster_values.isna().sum()
+                                else:
+                                    top_element_count = np.sum(cluster_values == top_element)
+                                top_element_percentage = top_element_count / cluster_values.size * 100
+                                hv_text_cluster += (
+                                    f"<br />{el} top: {format_missing_value(top_element)}"
+                                    f" ({top_element_percentage:.1f}%)"
                                 )
-                                hv_text_cluster += f"<br />{el} top: {top_element} ({top_element_percentage:.1f}%)"
                         mean_predicted_value = df_pred.loc[df_pred["cluster"] == c, "proba_values"].mean()
                         hv_text_cluster += f"<br />Mean predicted value: {mean_predicted_value:.{compute_digit_number(mean_predicted_value, 3)}f}"
                         if "error" in df_pred.columns:
@@ -2490,7 +2502,7 @@ def clustering_by_explainability_plot(
                 for idx, row in df_pred.iterrows():
                     text = f"Id: {idx}<br />"
                     if el not in ["predictions", "targets", "errors"]:
-                        text += f"{el}: {row[el]}<br />"
+                        text += f"{el}: {format_missing_value(row[el])}<br />"
                     text += f"Predicted Value: {row['predict_value']:.{self._round_digit}f}<br />"
                     if "error" in df_pred.columns:
                         text += f"Error: {row['error']:.{compute_digit_number(row['error'])}f}<br />"
@@ -2504,19 +2516,27 @@ def clustering_by_explainability_plot(
                     if el not in ["predictions", "targets", "errors"]:
                         is_num = is_numeric_dtype(df_pred[el]) and not is_bool_dtype(df_pred[el])
                         n_unique = df_pred[el].nunique(dropna=True)
+                        cluster_values = df_pred.loc[df_pred["cluster"] == c, el]
                         if is_num and n_unique > 5:
-                            mean_el = df_pred.loc[df_pred["cluster"] == c, el].mean()
-                            std_el = df_pred.loc[df_pred["cluster"] == c, el].std()
-                            hv_text_cluster += f"<br />{el} mean: {mean_el:.{compute_digit_number(mean_el, 3)}f}"
-                            hv_text_cluster += f"<br />{el} std: {std_el:.{compute_digit_number(std_el, 3)}f}"
+                            mean_el = cluster_values.mean()
+                            std_el = cluster_values.std()
+                            if pd.isna(mean_el) or pd.isna(std_el):
+                                hv_text_cluster += f"<br />{el} mean: {format_missing_value(mean_el)}"
+                                hv_text_cluster += f"<br />{el} std: {format_missing_value(std_el)}"
+                            else:
+                                hv_text_cluster += f"<br />{el} mean: {mean_el:.{compute_digit_number(mean_el, 3)}f}"
+                                hv_text_cluster += f"<br />{el} std: {std_el:.{compute_digit_number(std_el, 3)}f}"
                         else:
-                            top_element = df_pred.loc[df_pred["cluster"] == c, el].mode()[0]
-                            top_element_percentage = (
-                                np.sum(df_pred.loc[df_pred["cluster"] == c, el] == top_element)
-                                / df_pred.loc[df_pred["cluster"] == c, el].size
-                                * 100
+                            # dropna=False so that null values can be reported as the top modality
+                            top_element = cluster_values.mode(dropna=False).iloc[0]
+                            if pd.isna(top_element):
+                                top_element_count = cluster_values.isna().sum()
+                            else:
+                                top_element_count = np.sum(cluster_values == top_element)
+                            top_element_percentage = top_element_count / cluster_values.size * 100
+                            hv_text_cluster += (
+                                f"<br />{el} top: {format_missing_value(top_element)} ({top_element_percentage:.1f}%)"
                             )
-                            hv_text_cluster += f"<br />{el} top: {top_element} ({top_element_percentage:.1f}%)"
                     mean_predicted_value = df_pred.loc[df_pred["cluster"] == c, "predict_value"].mean()
                     hv_text_cluster += f"<br />Mean predicted value: {mean_predicted_value:.{compute_digit_number(mean_predicted_value, 3)}f}"
                     if "error" in df_pred.columns:

shapash/plots/plot_bar_chart.py

@@ -1,7 +1,7 @@
 from plotly import graph_objs as go
 from plotly.offline import plot
 
-from shapash.utils.utils import add_line_break, adjust_title_height, truncate_str
+from shapash.utils.utils import add_line_break, adjust_title_height, format_missing_value, truncate_str
 
 
 def plot_bar_chart(
@@ -103,6 +103,7 @@ def plot_bar_chart(
         bars = []
         for num, expl in enumerate(zip(var_dict, x_val, contrib, strict=False)):
             feat_name, x_val_el, contrib_value = expl
+            x_val_el = format_missing_value(x_val_el)
             is_grouped = False
             if x_val_el == "":
                 ylabel = f"<i>{feat_name}</i>"
@@ -116,7 +117,7 @@ def plot_bar_chart(
                     feat_groups_values = x_init[features_groups[group_name]].loc[index_value[0]]
                     hoverlabel = "<br />".join(
                         [
-                            f"<b>{add_line_break(features_dict.get(f_name, f_name), 40, maxlen=120)} :</b>{add_line_break(f_value, 40, maxlen=160)}"
+                            f"<b>{add_line_break(features_dict.get(f_name, f_name), 40, maxlen=120)} :</b>{add_line_break(format_missing_value(f_value), 40, maxlen=160)}"
                             for f_name, f_value in feat_groups_values.to_dict().items()
                         ]
                     )