diff --git a/envs/environment_a.yml b/envs/environment_a.yml index d8214313a9..f09513a3c4 100644 --- a/envs/environment_a.yml +++ b/envs/environment_a.yml @@ -10,7 +10,7 @@ dependencies: - iris=3.12 - joblib - numpy=2.2 - - pandas>=3 # https://github.com/SciTools/iris/issues/6761 + - pandas>=3 - proj=9.7.1 # https://github.com/metoppv/improver/issues/2325 - pytz - scipy=1.15 @@ -21,6 +21,7 @@ dependencies: - statsmodels - geopandas - numba + - pyarrow - pygam<0.11 - pysteps - python-stratify @@ -29,6 +30,7 @@ dependencies: - tensorflow-hub - scikit-learn - kmedoids + - xarray # Development - astroid - codacy-coverage @@ -42,7 +44,6 @@ dependencies: - sphinx-book-theme - sphinx-gallery - threadpoolctl - - xarray - pip - pip: - git+https://github.com/metoppv/improver_example_data.git@${BRANCH_NAME} diff --git a/envs/latest.yml b/envs/latest.yml index e33ce1022d..f5a881bfaf 100644 --- a/envs/latest.yml +++ b/envs/latest.yml @@ -14,6 +14,7 @@ dependencies: - iris - joblib - numpy + - pandas - pytz - scipy - sphinx<9.0.0 # https://github.com/tox-dev/sphinx-autodoc-typehints/issues/586 @@ -24,6 +25,7 @@ dependencies: - quantile-forest - lightgbm - numba + - pyarrow - pygam<0.11 - pysteps - treelite @@ -37,7 +39,6 @@ dependencies: - filelock - mock - mypy - - pandas<3 # https://github.com/SciTools/iris/issues/6761 - proj=9.7.1 # https://github.com/metoppv/improver/issues/2325 - pytest - pytest-cov diff --git a/improver/calibration/__init__.py b/improver/calibration/__init__.py index a3633de7ee..7f1fc21aa1 100644 --- a/improver/calibration/__init__.py +++ b/improver/calibration/__init__.py @@ -38,18 +38,18 @@ def __init__(self): ("percentile", pa.float64()), ("forecast", pa.float32()), ("altitude", pa.float32()), - ("blend_time", pa.timestamp("us", "utc")), - ("forecast_period", pa.duration("us")), - ("forecast_reference_time", pa.timestamp("us", "utc")), + ("blend_time", pa.timestamp("s", "utc")), + ("forecast_period", pa.duration("s")), + ("forecast_reference_time", pa.timestamp("s", "utc")), ("latitude", pa.float32()), ("longitude", pa.float32()), - ("time", pa.timestamp("us", "utc")), + ("time", pa.timestamp("s", "utc")), ("wmo_id", pa.string()), ("station_id", pa.string()), ("cf_name", pa.string()), ("units", pa.string()), ("experiment", pa.string()), - ("period", pa.duration("us")), + ("period", pa.duration("s")), ("height", pa.float32()), ("diagnostic", pa.string()), ] @@ -60,7 +60,7 @@ def __init__(self): ("latitude", pa.float32()), ("longitude", pa.float32()), ("altitude", pa.float32()), - ("time", pa.timestamp("us", "utc")), + ("time", pa.timestamp("s", "utc")), ("wmo_id", pa.string()), ("station_id", pa.string()), ("ob_value", pa.float32()), diff --git a/improver/calibration/load_and_train_quantile_regression_random_forest.py b/improver/calibration/load_and_train_quantile_regression_random_forest.py index 54731ba46b..6f2aa1ea7f 100644 --- a/improver/calibration/load_and_train_quantile_regression_random_forest.py +++ b/improver/calibration/load_and_train_quantile_regression_random_forest.py @@ -284,10 +284,9 @@ def _read_parquet_files( merge_on, ) - seconds_to_us = 1e6 forecast_df = forecast_df[ forecast_df["forecast_period"].isin( - pd.to_timedelta(np.array(forecast_periods) * seconds_to_us, unit="us") + pd.to_timedelta(np.array(forecast_periods), unit="s") ) ].reset_index(drop=True) diff --git a/improver_tests/calibration/quantile_regression_random_forests_calibration/test_load_and_train_quantile_regression_random_forest.py b/improver_tests/calibration/quantile_regression_random_forests_calibration/test_load_and_train_quantile_regression_random_forest.py index 10f55caad2..61ebeb95ab 100644 --- a/improver_tests/calibration/quantile_regression_random_forests_calibration/test_load_and_train_quantile_regression_random_forest.py +++ b/improver_tests/calibration/quantile_regression_random_forests_calibration/test_load_and_train_quantile_regression_random_forest.py @@ -449,13 +449,13 @@ def filter_forecast_periods(forecast_df, forecast_periods): else: all_periods.append(int(token)) - # Convert to seconds and deduplicate + # Convert to seconds and deduplicate. forecast_periods_s = [fp * 3600 for fp in sorted(set(all_periods))] - target_periods = pd.to_timedelta(np.array(forecast_periods_s) * 1e6, unit="us") + target_periods = pd.to_timedelta(np.array(forecast_periods_s), unit="s") forecast_df = forecast_df.copy() forecast_df["forecast_period"] = pd.to_timedelta( - forecast_df["forecast_period"], unit="us" + forecast_df["forecast_period"], unit="s" ) return forecast_df[forecast_df["forecast_period"].isin(target_periods)].reset_index( @@ -475,10 +475,10 @@ def amend_expected_forecast_df( forecast_df = filter_forecast_periods(forecast_df, forecast_periods) for column in ["time", "forecast_reference_time", "blend_time"]: forecast_df[column] = pd.to_datetime(forecast_df[column], utc=True) - forecast_df[column] = forecast_df[column].astype("datetime64[us, UTC]") + forecast_df[column] = forecast_df[column].astype("datetime64[s, UTC]") for column in ["forecast_period", "period"]: - forecast_df[column] = pd.to_timedelta(forecast_df[column], unit="us") - forecast_df[column] = forecast_df[column].astype("timedelta64[us]") + forecast_df[column] = pd.to_timedelta(forecast_df[column], unit="s") + forecast_df[column] = forecast_df[column].astype("timedelta64[s]") base_df = forecast_df[forecast_df["diagnostic"] == parquet_diagnostic_names[0]] for parquet_diagnostic_name, cf_name in zip( @@ -519,7 +519,7 @@ def amend_expected_truth_df(truth_df, parquet_diagnostic_name): """Amend the expected truth DataFrame to match the output of the plugin.""" truth_df = truth_df[truth_df["diagnostic"] == parquet_diagnostic_name] truth_df["time"] = pd.to_datetime(truth_df["time"], utc=True) - truth_df["time"] = truth_df["time"].astype("datetime64[us, UTC]") + truth_df["time"] = truth_df["time"].astype("datetime64[s, UTC]") return truth_df @@ -1282,7 +1282,7 @@ def test_unexpected_preparation( truth_df["time"] = pd.Series( [pd.Timestamp("2020-01-01 00:00:00", tz="utc")] * len(truth_df), index=truth_df.index, - ).astype("datetime64[us, UTC]") + ).astype("datetime64[s, UTC]") plugin = PrepareAndTrainQRF( feature_config=feature_config,