Switch to using seconds as the units

envs/environment_a.yml

@@ -10,7 +10,7 @@ dependencies:
   - iris=3.12
   - joblib
   - numpy=2.2
-  - pandas>=3  # https://github.com/SciTools/iris/issues/6761
+  - pandas>=3
   - proj=9.7.1  # https://github.com/metoppv/improver/issues/2325
   - pytz
   - scipy=1.15
@@ -21,6 +21,7 @@ dependencies:
   - statsmodels
   - geopandas
   - numba
+  - pyarrow
   - pygam<0.11
   - pysteps
   - python-stratify
@@ -29,6 +30,7 @@ dependencies:
   - tensorflow-hub
   - scikit-learn
   - kmedoids
+  - xarray
   # Development
   - astroid
   - codacy-coverage
@@ -42,7 +44,6 @@ dependencies:
   - sphinx-book-theme  
   - sphinx-gallery
   - threadpoolctl
-  - xarray
   - pip
   - pip:
     - git+https://github.com/metoppv/improver_example_data.git@${BRANCH_NAME} 

envs/latest.yml

@@ -14,6 +14,7 @@ dependencies:
   - iris
   - joblib
   - numpy
+  - pandas
   - pytz
   - scipy
   - sphinx<9.0.0  # https://github.com/tox-dev/sphinx-autodoc-typehints/issues/586
@@ -24,6 +25,7 @@ dependencies:
   - quantile-forest
   - lightgbm
   - numba
+  - pyarrow
   - pygam<0.11
   - pysteps
   - treelite
@@ -37,7 +39,6 @@ dependencies:
   - filelock
   - mock
   - mypy
-  - pandas<3  # https://github.com/SciTools/iris/issues/6761
   - proj=9.7.1  # https://github.com/metoppv/improver/issues/2325
   - pytest
   - pytest-cov

improver/calibration/__init__.py

@@ -38,18 +38,18 @@ def __init__(self):
                 ("percentile", pa.float64()),
                 ("forecast", pa.float32()),
                 ("altitude", pa.float32()),
-                ("blend_time", pa.timestamp("us", "utc")),
-                ("forecast_period", pa.duration("us")),
-                ("forecast_reference_time", pa.timestamp("us", "utc")),
+                ("blend_time", pa.timestamp("s", "utc")),
+                ("forecast_period", pa.duration("s")),
+                ("forecast_reference_time", pa.timestamp("s", "utc")),
                 ("latitude", pa.float32()),
                 ("longitude", pa.float32()),
-                ("time", pa.timestamp("us", "utc")),
+                ("time", pa.timestamp("s", "utc")),
                 ("wmo_id", pa.string()),
                 ("station_id", pa.string()),
                 ("cf_name", pa.string()),
                 ("units", pa.string()),
                 ("experiment", pa.string()),
-                ("period", pa.duration("us")),
+                ("period", pa.duration("s")),
                 ("height", pa.float32()),
                 ("diagnostic", pa.string()),
             ]
@@ -60,7 +60,7 @@ def __init__(self):
                 ("latitude", pa.float32()),
                 ("longitude", pa.float32()),
                 ("altitude", pa.float32()),
-                ("time", pa.timestamp("us", "utc")),
+                ("time", pa.timestamp("s", "utc")),
                 ("wmo_id", pa.string()),
                 ("station_id", pa.string()),
                 ("ob_value", pa.float32()),

improver/calibration/load_and_train_quantile_regression_random_forest.py

@@ -284,10 +284,9 @@ def _read_parquet_files(
                 merge_on,
             )
 
-        seconds_to_us = 1e6
         forecast_df = forecast_df[
             forecast_df["forecast_period"].isin(
-                pd.to_timedelta(np.array(forecast_periods) * seconds_to_us, unit="us")
+                pd.to_timedelta(np.array(forecast_periods), unit="s")
             )
         ].reset_index(drop=True)
 

improver_tests/calibration/quantile_regression_random_forests_calibration/test_load_and_train_quantile_regression_random_forest.py

@@ -449,13 +449,13 @@ def filter_forecast_periods(forecast_df, forecast_periods):
         else:
             all_periods.append(int(token))
 
-    # Convert to seconds and deduplicate
+    # Convert to seconds and deduplicate.
     forecast_periods_s = [fp * 3600 for fp in sorted(set(all_periods))]
-    target_periods = pd.to_timedelta(np.array(forecast_periods_s) * 1e6, unit="us")
+    target_periods = pd.to_timedelta(np.array(forecast_periods_s), unit="s")
 
     forecast_df = forecast_df.copy()
     forecast_df["forecast_period"] = pd.to_timedelta(
-        forecast_df["forecast_period"], unit="us"
+        forecast_df["forecast_period"], unit="s"
     )
 
     return forecast_df[forecast_df["forecast_period"].isin(target_periods)].reset_index(
@@ -475,10 +475,10 @@ def amend_expected_forecast_df(
     forecast_df = filter_forecast_periods(forecast_df, forecast_periods)
     for column in ["time", "forecast_reference_time", "blend_time"]:
         forecast_df[column] = pd.to_datetime(forecast_df[column], utc=True)
-        forecast_df[column] = forecast_df[column].astype("datetime64[us, UTC]")
+        forecast_df[column] = forecast_df[column].astype("datetime64[s, UTC]")
     for column in ["forecast_period", "period"]:
-        forecast_df[column] = pd.to_timedelta(forecast_df[column], unit="us")
-        forecast_df[column] = forecast_df[column].astype("timedelta64[us]")
+        forecast_df[column] = pd.to_timedelta(forecast_df[column], unit="s")
+        forecast_df[column] = forecast_df[column].astype("timedelta64[s]")
 
     base_df = forecast_df[forecast_df["diagnostic"] == parquet_diagnostic_names[0]]
     for parquet_diagnostic_name, cf_name in zip(
@@ -519,7 +519,7 @@ def amend_expected_truth_df(truth_df, parquet_diagnostic_name):
     """Amend the expected truth DataFrame to match the output of the plugin."""
     truth_df = truth_df[truth_df["diagnostic"] == parquet_diagnostic_name]
     truth_df["time"] = pd.to_datetime(truth_df["time"], utc=True)
-    truth_df["time"] = truth_df["time"].astype("datetime64[us, UTC]")
+    truth_df["time"] = truth_df["time"].astype("datetime64[s, UTC]")
     return truth_df
 
 
@@ -1282,7 +1282,7 @@ def test_unexpected_preparation(
     truth_df["time"] = pd.Series(
         [pd.Timestamp("2020-01-01 00:00:00", tz="utc")] * len(truth_df),
         index=truth_df.index,
-    ).astype("datetime64[us, UTC]")
+    ).astype("datetime64[s, UTC]")
 
     plugin = PrepareAndTrainQRF(
         feature_config=feature_config,