25956: Fixes an errant preserve_rare_values warning, hones IFA dtype validation/bugfixes, cleans up precision warnings

howso/utilities/feature_attributes/base.py

@@ -376,12 +376,22 @@ def _validate_bounds(self, data: pd.DataFrame, feature: str,
                 f'"{feature}" had {additional_errors} additional values outside of bounds that were not displayed.')
         return errors
 
+    @staticmethod
+    def _is_numeric_dtype(dtype: str | np.dtype | pd.api.extensions.ExtensionDtype | pd.CategoricalDtype) -> bool:
+        """Return whether `dtype` holds numbers, i.e. an integer, nullable integer, or float dtype."""
+        try:
+            dtype = pd.api.types.pandas_dtype(dtype)
+        except TypeError:
+            return False
+        return pd.api.types.is_numeric_dtype(dtype) and not pd.api.types.is_bool_dtype(dtype)
+
     def _validate_dtype(self, data: pd.DataFrame, feature: str,
                         expected_dtype: str | pd.CategoricalDtype, coerced_df: pd.DataFrame,
                         coerce: bool = False, localize_datetimes: bool = True) -> list[str]:
         """Validate the data type of a feature and optionally attempt to coerce."""
         errors = []
         series = coerced_df[feature]
+        actual_dtype = data[feature].dtype
         is_valid = False
         coerce_err = ""
 
@@ -392,26 +402,26 @@ def _validate_dtype(self, data: pd.DataFrame, feature: str,
                 if coerce:
                     coerced_df[feature] = series
                 is_valid = True
-            except Exception: # noqa: Intentionally broad
-                pass
+            except Exception as err: # noqa: Intentionally broad
+                coerce_err = str(err)
         elif expected_dtype == "datetime64":
             try:
                 format = self[feature]["date_time_format"]  # pyright: ignore[reportTypedDictNotRequiredAccess]
                 if ".%f" in format:
                     format = "ISO8601"
                 series = pd.to_datetime(coerced_df[feature], format=format)
                 if coerce:
-                    if localize_datetimes and not isinstance(series, pd.DatetimeTZDtype):
+                    if localize_datetimes and not isinstance(series.dtype, pd.DatetimeTZDtype):
                         coerced_df[feature] = series.dt.tz_localize(
                             "UTC", ambiguous="infer", nonexistent="NaT"
                         )
                     else:
                         coerced_df[feature] = series
                 is_valid = True
-            except Exception: # noqa: Intentionally broad
-                pass
+            except Exception as err: # noqa: Intentionally broad
+                coerce_err = str(err)
         # Else, compare the dtype directly
-        elif data[feature].dtype.name == expected_dtype:
+        elif actual_dtype.name == expected_dtype:
             is_valid = True
         # If the feature can be converted, consider it valid (slightly differing numeric types, etc.)
         else:
@@ -420,22 +430,22 @@ def _validate_dtype(self, data: pd.DataFrame, feature: str,
                 if coerce:
                     coerced_df[feature] = series
                 is_valid = True
-            except pd.errors.IntCastingNaNError:
-                # If this happens, there is a null value, thus a float dtype is OK
-                if pd.api.types.is_float_dtype(series):
-                    is_valid = True
             except Exception as err: # noqa: Intentionally broad
+                # Numeric dtypes differ only in representation here: validation does not alter the
+                # data unless `coerce` is set, so a numeric column is trained as it stands whichever
+                # dtype the attributes imply. A column that cannot be cast keeps its own dtype.
+                is_valid = self._is_numeric_dtype(expected_dtype) and self._is_numeric_dtype(actual_dtype)
                 coerce_err = str(err)
 
         # Raise warnings if the types do not match
         if not is_valid:
             if coerce:
                 errors.append(f"Expected dtype '{expected_dtype}' for feature '{feature}' "
-                              f"but could not coerce:\nActual dtype: {data[feature].dtype}"
+                              f"but could not coerce:\nActual dtype: {actual_dtype}"
                               f"\nError raised from Pandas.astype():\n\n{coerce_err}")
             else:
                 errors.append(f"Feature '{feature}' should be '{expected_dtype}' dtype, but found "
-                              f"'{data[feature].dtype}'")
+                              f"'{actual_dtype}'")
 
         return errors
 
@@ -482,9 +492,10 @@ def _validate_df(self, data: pd.DataFrame, coerce: bool = False,
                         errors.extend(self._validate_dtype(data, feature, "int64",
                                                            coerced_df, coerce=coerce))
                 elif attributes.get("data_type") == "boolean":
-                    # Check type (boolean)
-                    errors.extend(self._validate_dtype(data, feature, "bool",
-                                                       coerced_df, coerce=coerce))
+                    # Check type (boolean). A boolean column that also holds nulls stays an object
+                    # column, since casting it to `bool` would turn every null into `False`.
+                    errors.extend(self._validate_dtype(data, feature, "bool", coerced_df,
+                                                       coerce=coerce and not data[feature].isna().any()))
                 elif attributes.get("bounds") and attributes["bounds"].get("allowed"):  # pyright: ignore[reportTypedDictNotRequiredAccess]
                     # Check type (categorical)
                     schema_dtype = pd.CategoricalDtype(attributes["bounds"]["allowed"],  # pyright: ignore[reportTypedDictNotRequiredAccess]
@@ -2026,8 +2037,8 @@ def _process_rare_values(  # noqa: PLR0912, PLR0915
             max_distilled_cases, _ = get_optimized_max_chunk_size(row_count=self._get_row_count(),
                                                                   max_chunk_size=max_distilled_cases)
         else:
-            # Set a small default
-            max_distilled_cases = 25_000
+            # Set a small default; keep consistent with Enterprise
+            max_distilled_cases = 50_000
 
         # Workflow 1: User provided a config with protected multipliers; may need to compute unprotected multipliers
         if preserve_rare_values_config is not None:

howso/utilities/feature_attributes/pandas.py

@@ -783,23 +783,18 @@ def _infer_floating_point_attributes(self, feature_name: str) -> dict:
                 for r in col_array
             ])
 
-            # specify decimal place. Proceed with training but issue a warning.
+            # Specify decimal places for features the engine can represent exactly. Features beyond
+            # that precision are trained without the attribute and reported in a single warning.
             if pd.api.types.is_float_dtype(col.dtype):
-                try:
-                    if getattr(col.dtype, 'itemsize') <= 8:
-                        attributes['decimal_places'] = decimals
-                    else:
-                        warnings.warn(
-                            f'Feature "{feature_name}" contains floating point '
-                            'values that exceed the maximum supported precision '
-                            'of 64 bits.'
-                        )
-                except AttributeError:
-                    warnings.warn(
-                        f'Feature "{feature_name}" may contain floating point '
-                        'values that exceed the maximum supported precision '
-                        'of 64 bits.'
-                    )
+                item_size = getattr(col.dtype, 'itemsize', None)
+                if item_size is None:
+                    self.warnings_collector.triage(
+                        IFAWarningEmitterType.POSSIBLE_EXCESSIVE_FLOAT_PRECISION, feature_name)
+                elif item_size <= 8:
+                    attributes['decimal_places'] = decimals
+                else:
+                    self.warnings_collector.triage(
+                        IFAWarningEmitterType.EXCESSIVE_FLOAT_PRECISION, feature_name)
 
         return attributes
 

howso/utilities/feature_attributes/suggestions.py

@@ -342,55 +342,52 @@ def summary(self) -> str:
         return (f"Found {_count(num_values, 'rare value')} across {_count(len(self._prvc), 'column')} "
                 "whose signal may be lost during data distillation workflows")
 
-    def apply(self, attributes: dict) -> None:
+    def _warn_default_max_distilled_cases(self, addendum: str = "", stack_level: int = 4) -> None:
+        """
+        Warn that the case weight multipliers were computed from a default ``max_distilled_cases``.
+
+        Parameters
+        ----------
+        addendum : str, default ""
+            An additional sentence appended to the warning, describing the consequence for the
+            calling method.
+        stack_level : int, default 4
+            The stack level value to pass into `warn` via `stacklevel`. The default attributes the
+            warning to the caller of `apply_suggestion()`; methods a user calls directly pass 3.
+        """
+        warnings.warn(
+            "The computed case weights for rare value multipliers are likely inaccurate as "
+            "`max_distilled_cases` was not provided to `infer_feature_attributes`. Please provide "
+            "this parameter or be aware that the case weight multipliers were computed based on a "
+            "default `max_distilled_cases` value of 50,000. "
+            "An accurate `max_distilled_cases` enables Howso to correctly weight the influence of rare "
+            "values in the data, since the weighting is calibrated proportionally to the number of cases "
+            "remaining after distillation." + addendum,
+            UserWarning,
+            stacklevel=stack_level,
+        )
+
+    def apply(self, attributes: Mapping[str, Any]) -> None:
         """Apply the computed rare values preservation config to the FeatureAttributesBase object."""
         if not self._user_set_mdc:
-            warnings.warn(
-                "The computed case weights for Rare values multipliers  are likely inaccurate as "
-                "`max_distilled_cases` was not provided to `infer_feature_attributes`. Please provide "
-                "this parameter or be aware that the case weight multipliers were computed based on a "
-                "default `max_distilled_cases` value of 25,000. "
-                "An accurate max_distilled_cases enables Howso to correctly weight the influence of rare "
-                "values in the data, since the weighting is calibrated proportionally to the number of cases "
-                "remaining after distillation. Since an inaccurate value may result in rare values being "
-                "under-weighted or over-weighted, this suggestion was not applied.",
-                UserWarning,
-                stacklevel=3,
+            self._warn_default_max_distilled_cases(
+                " Since an inaccurate value may result in rare values being under-weighted or "
+                "over-weighted, this suggestion was not applied."
             )
-        if self._user_set_mdc:
-            for feature, config in self._prvc.items():
-                attributes[feature]["preserve_rare_values"] = config
+            return
+        for feature, config in self._prvc.items():
+            attributes[feature]["preserve_rare_values"] = config
 
-    def get_config(self) -> FullPreserveRareValuesConfig:
+    def get_config(self, enable_warnings: bool = True) -> FullPreserveRareValuesConfig:
         """Get the `preserve_rare_values_config` for use in future calls to `infer_feature_attributes`."""
-        if not self._user_set_mdc:
-            warnings.warn(
-                "The computed case weights for Rare values multipliers  are likely inaccurate as "
-                "`max_distilled_cases` was not provided to `infer_feature_attributes`. Please provide "
-                "this parameter or be aware that the case weight multipliers were computed based on a "
-                "default `max_distilled_cases` value of 25,000. "
-                "An accurate max_distilled_cases enables Howso to correctly weight the influence of rare "
-                "values in the data, since the weighting is calibrated proportionally to the number of cases "
-                "remaining after distillation.",
-                UserWarning,
-                stacklevel=3,
-            )
+        if not self._user_set_mdc and enable_warnings:
+            self._warn_default_max_distilled_cases(stack_level=3)
         return self._prvc
 
     def get_values_map(self) -> PreserveRareValuesMap:
         """Get the `preserve_rare_values_map` for use in future calls to `infer_feature_attributes."""
         if not self._user_set_mdc:
-            warnings.warn(
-                "The computed case weights for Rare values multipliers  are likely inaccurate as "
-                "`max_distilled_cases` was not provided to `infer_feature_attributes`. Please provide "
-                "this parameter or be aware that the case weight multipliers were computed based on a "
-                "default `max_distilled_cases` value of 25,000. "
-                "An accurate max_distilled_cases enables Howso to correctly weight the influence of rare "
-                "values in the data, since the weighting is calibrated proportionally to the number of cases "
-                "remaining after distillation.",
-                UserWarning,
-                stacklevel=3,
-            )
+            self._warn_default_max_distilled_cases(stack_level=3)
         values_map = {}
         for feature, config in self._prvc.items():
             multipliers = config["protected_values_multipliers"]
@@ -401,7 +398,7 @@ def merge(self, other: IFASuggestion) -> None:
         """Merge another PRVSuggestion into this one if there are no conflicts."""
         if not isinstance(other, PRVSuggestion):
             raise TypeError(f"Cannot merge {type(other).__name__} into PRVSuggestion.")
-        for feature, config in other.get_config().items():
+        for feature, config in other.get_config(enable_warnings=False).items():
             if feature not in self._prvc:
                 self._prvc[feature] = config
             elif self._prvc[feature] != config:

howso/utilities/feature_attributes/tests/test_infer_feature_attributes.py

@@ -308,6 +308,36 @@ def test_get_feature_type_raises(data, data_type):
         infer_feature_attributes(df)
 
 
+def test_excessive_float_precision_warning():
+    """Test that features exceeding 64-bit float precision are reported in a single warning."""
+    # Place this here to avoid circular import
+    from howso.utilities.feature_attributes.pandas import InferFeatureAttributesDataFrame
+    if not hasattr(np, "float128"):
+        pytest.skip("Unsupported platform")
+
+    df = pd.DataFrame({
+        "a": np.arange(20, dtype=np.float128) + 0.5,
+        "b": np.arange(20, dtype=np.float128) * 1.5,
+        "c": np.arange(20, dtype="float64") + 0.25,
+    })
+    ifa = InferFeatureAttributesDataFrame(df)
+    ifa.attributes = {}
+    attributes = {feature: ifa._infer_floating_point_attributes(feature) for feature in df.columns}
+
+    # Features beyond the supported precision get no `decimal_places`
+    assert "decimal_places" not in attributes["a"]
+    assert "decimal_places" not in attributes["b"]
+    assert attributes["c"]["decimal_places"] == 2
+
+    with pytest.warns(UserWarning, match="exceed the maximum supported precision") as record:
+        ifa.warnings_collector.emit_all()
+
+    assert len(record) == 1
+    message = str(record[0].message)
+    assert "- a" in message and "- b" in message
+    assert "- c" not in message
+
+
 @pytest.mark.parametrize("should_fail, data", [
     (True, [[1]]),
     (True, {3: [1]}),

howso/utilities/feature_attributes/tests/test_warnings.py

@@ -11,7 +11,31 @@ def test_warnings_emitters():
     collector.triage(IFAWarningEmitterType.MISSING_TZ_FEATURES, "b")
     collector.triage(IFAWarningEmitterType.UNKNOWN_DATETIME_FORMAT, "c")
     collector.triage(IFAWarningEmitterType.UTC_OFFSET, "d")
+    collector.triage(IFAWarningEmitterType.EXCESSIVE_FLOAT_PRECISION, "e")
+    collector.triage(IFAWarningEmitterType.POSSIBLE_EXCESSIVE_FLOAT_PRECISION, "f")
 
-    with pytest.warns(UserWarning, match=r"- [a-d]") as record:
+    with pytest.warns(UserWarning, match=r"- [a-f]") as record:
         collector.emit_all()
-        assert len(record) == 4
+        assert len(record) == 6
+
+
+# `SIMPLE` collects whole messages rather than feature names, so it emits one warning per message.
+@pytest.mark.parametrize("emitter_type", [t for t in IFAWarningEmitterType if t != IFAWarningEmitterType.SIMPLE])
+def test_warnings_emitters_list_all_features(emitter_type):
+    """Test that features sharing an emitter are listed in a single warning."""
+    collector = IFAWarningCollector()
+    for feature in ("a", "b", "c"):
+        collector.triage(emitter_type, feature)
+
+    with pytest.warns(UserWarning) as record:
+        collector.emit_all()
+
+    assert len(record) == 1
+    message = str(record[0].message)
+    assert all(feature in message for feature in ("a", "b", "c"))
+
+
+def test_warnings_emitters_unknown_type():
+    """Test that an unknown emitter type is rejected."""
+    with pytest.raises(ValueError, match="Unknown `emitter_type` provided."):
+        IFAWarningCollector().triage("not_an_emitter_type", "a")