Fix pandas 3.1 deprecations, require pandas>=2.3

* Don't pass `float_precision` to `read_csv` for pandas>=3.1, where
  round-trip float parsing is the default
* Replace deprecated `inplace=True` by reassignment. As a consequence,
  `get_{condition,observable,parameter,mapping}_df` no longer modify
  DataFrames passed by the caller
* Avoid `fillna` with incompatible fill values
* Use a full-length key when adding the sum row to a MultiIndex
  DataFrame in `get_data_per_observable`
* Treat pandas deprecation warnings as errors in tests. They derive from
  `DeprecationWarning`, which is ignored in `pytest.ini`
* Require pandas>=2.3 in accordance with NEP 29

Closes #570

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

Author: dweindl

conftest.py

@@ -0,0 +1,13 @@
+"""pytest configuration."""
+
+import pandas as pd
+
+
+def pytest_configure(config):
+    # pandas deprecation warnings are subclasses of DeprecationWarning,
+    # which is ignored in pytest.ini. Treat them as errors.
+    # PandasChangeWarning is only available in pandas>=3.0.
+    if hasattr(pd.errors, "PandasChangeWarning"):
+        config.addinivalue_line(
+            "filterwarnings", "error::pandas.errors.PandasChangeWarning"
+        )

petab/_utils.py

@@ -2,10 +2,20 @@
 
 from pathlib import Path
 
+import pandas as pd
 from pydantic import AnyUrl, TypeAdapter
 
 PathOrUrlAdapter = TypeAdapter(AnyUrl | Path)
 
+#: Extra keyword arguments for :func:`pandas.read_csv` to ensure lossless
+#: float parsing. ``float_precision`` is deprecated in pandas>=3.1,
+#: where round-trip parsing is the default.
+READ_CSV_FLOAT_KWARGS = (
+    {"float_precision": "round_trip"}
+    if tuple(map(int, pd.__version__.split(".")[:2])) < (3, 1)
+    else {}
+)
+
 
 def _generate_path(
     file_path: str | Path | AnyUrl,

petab/v1/conditions.py

@@ -6,6 +6,7 @@
 import numpy as np
 import pandas as pd
 
+from .._utils import READ_CSV_FLOAT_KWARGS
 from . import core, lint
 from .C import *
 
@@ -32,21 +33,20 @@ def get_condition_df(
 
     if isinstance(condition_file, str | Path):
         condition_file = pd.read_csv(
-            condition_file, sep="\t", float_precision="round_trip"
+            condition_file, sep="\t", **READ_CSV_FLOAT_KWARGS
         )
 
     lint.assert_no_leading_trailing_whitespace(
         condition_file.columns.values, "condition"
     )
 
     if not isinstance(condition_file.index, pd.RangeIndex):
-        condition_file.reset_index(
+        condition_file = condition_file.reset_index(
             drop=condition_file.index.name != CONDITION_ID,
-            inplace=True,
         )
 
     try:
-        condition_file.set_index([CONDITION_ID], inplace=True)
+        condition_file = condition_file.set_index([CONDITION_ID])
     except KeyError:
         raise KeyError(
             f"Condition table missing mandatory field {CONDITION_ID}."
@@ -90,7 +90,7 @@ def create_condition_df(
             raise ValueError("Invalid parameter ID: " + p)
         df[p] = np.nan
 
-    df.set_index(CONDITION_ID, inplace=True)
+    df = df.set_index(CONDITION_ID)
 
     return df
 

petab/v1/core.py

@@ -14,6 +14,7 @@
 import pandas as pd
 from pandas.api.types import is_string_dtype
 
+from .._utils import READ_CSV_FLOAT_KWARGS
 from . import yaml
 from .C import *  # noqa: F403
 
@@ -52,7 +53,7 @@ def get_simulation_df(simulation_file: str | Path) -> pd.DataFrame:
         Simulation DataFrame
     """
     return pd.read_csv(
-        simulation_file, sep="\t", index_col=None, float_precision="round_trip"
+        simulation_file, sep="\t", index_col=None, **READ_CSV_FLOAT_KWARGS
     )
 
 
@@ -94,7 +95,7 @@ def get_visualization_df(
             sep="\t",
             index_col=None,
             converters=types,
-            float_precision="round_trip",
+            **READ_CSV_FLOAT_KWARGS,
         )
     except pd.errors.EmptyDataError:
         warn(

petab/v1/mapping.py

@@ -5,6 +5,7 @@
 
 import pandas as pd
 
+from .._utils import READ_CSV_FLOAT_KWARGS
 from . import lint
 from .C import *  # noqa: F403
 from .models import Model
@@ -34,13 +35,12 @@ def get_mapping_df(
 
     if isinstance(mapping_file, str | Path):
         mapping_file = pd.read_csv(
-            mapping_file, sep="\t", float_precision="round_trip"
+            mapping_file, sep="\t", **READ_CSV_FLOAT_KWARGS
         )
 
     if not isinstance(mapping_file.index, pd.RangeIndex):
-        mapping_file.reset_index(
+        mapping_file = mapping_file.reset_index(
             drop=mapping_file.index.name != PETAB_ENTITY_ID,
-            inplace=True,
         )
 
     for col in MAPPING_DF_REQUIRED_COLS:
@@ -51,7 +51,7 @@ def get_mapping_df(
             mapping_file.reset_index()[col].values, col
         )
 
-    mapping_file.set_index([PETAB_ENTITY_ID], inplace=True)
+    mapping_file = mapping_file.set_index([PETAB_ENTITY_ID])
 
     return mapping_file
 

petab/v1/measurements.py

@@ -9,6 +9,7 @@
 import numpy as np
 import pandas as pd
 
+from .._utils import READ_CSV_FLOAT_KWARGS
 from . import core, lint, observables
 from .C import *  # noqa: F403
 
@@ -43,7 +44,7 @@ def get_measurement_df(
 
     if isinstance(measurement_file, str | Path):
         measurement_file = pd.read_csv(
-            measurement_file, sep="\t", float_precision="round_trip"
+            measurement_file, sep="\t", **READ_CSV_FLOAT_KWARGS
         )
 
     lint.assert_no_leading_trailing_whitespace(
@@ -95,7 +96,8 @@ def get_simulation_conditions(measurement_df: pd.DataFrame) -> pd.DataFrame:
     # We require NaN-containing rows, but they are ignored by `groupby`,
     # therefore replace them before
     simulation_conditions = (
-        measurement_df.fillna("")
+        measurement_df[grouping_cols]
+        .fillna("")
         .groupby(grouping_cols)
         .size()
         .reset_index()[grouping_cols]
@@ -130,7 +132,9 @@ def get_rows_for_condition(
     # check for equality in all grouping cols
     if PREEQUILIBRATION_CONDITION_ID in condition:
         row_filter = (
-            measurement_df[PREEQUILIBRATION_CONDITION_ID].fillna("")
+            measurement_df[PREEQUILIBRATION_CONDITION_ID]
+            .astype("object")
+            .fillna("")
             == condition[PREEQUILIBRATION_CONDITION_ID]
         ) & row_filter
     if SIMULATION_CONDITION_ID in condition:
@@ -252,7 +256,7 @@ def measurements_have_replicates(measurement_df: pd.DataFrame) -> bool:
         ],
     )
     return np.any(
-        measurement_df.fillna("").groupby(grouping_cols).size().values - 1
+        measurement_df.groupby(grouping_cols, dropna=False).size().values - 1
     )
 
 

petab/v1/observables.py

@@ -7,6 +7,7 @@
 
 import pandas as pd
 
+from .._utils import READ_CSV_FLOAT_KWARGS
 from . import core, lint
 from .C import *  # noqa: F403
 from .math import sympify_petab
@@ -39,21 +40,20 @@ def get_observable_df(
 
     if isinstance(observable_file, str | Path):
         observable_file = pd.read_csv(
-            observable_file, sep="\t", float_precision="round_trip"
+            observable_file, sep="\t", **READ_CSV_FLOAT_KWARGS
         )
 
     lint.assert_no_leading_trailing_whitespace(
         observable_file.columns.values, "observable"
     )
 
     if not isinstance(observable_file.index, pd.RangeIndex):
-        observable_file.reset_index(
+        observable_file = observable_file.reset_index(
             drop=observable_file.index.name != OBSERVABLE_ID,
-            inplace=True,
         )
 
     try:
-        observable_file.set_index([OBSERVABLE_ID], inplace=True)
+        observable_file = observable_file.set_index([OBSERVABLE_ID])
     except KeyError:
         raise KeyError(
             f"Observable table missing mandatory field {OBSERVABLE_ID}."

petab/v1/parameters.py

@@ -13,6 +13,7 @@
 import numpy as np
 import pandas as pd
 
+from .._utils import READ_CSV_FLOAT_KWARGS
 from . import conditions, core, lint, measurements, observables
 from .C import *  # noqa: F403
 from .models import Model
@@ -58,7 +59,7 @@ def get_parameter_df(
         parameter_df = parameter_file
     elif isinstance(parameter_file, str | Path):
         parameter_df = pd.read_csv(
-            parameter_file, sep="\t", float_precision="round_trip"
+            parameter_file, sep="\t", **READ_CSV_FLOAT_KWARGS
         )
     elif isinstance(parameter_file, Iterable):
         dfs = [get_parameter_df(x) for x in parameter_file if x]
@@ -77,13 +78,12 @@ def get_parameter_df(
     )
 
     if not isinstance(parameter_df.index, pd.RangeIndex):
-        parameter_df.reset_index(
+        parameter_df = parameter_df.reset_index(
             drop=parameter_file.index.name != PARAMETER_ID,
-            inplace=True,
         )
 
     try:
-        parameter_df.set_index([PARAMETER_ID], inplace=True)
+        parameter_df = parameter_df.set_index([PARAMETER_ID])
     except KeyError as e:
         raise KeyError(
             f"Parameter table missing mandatory field {PARAMETER_ID}."
@@ -239,7 +239,7 @@ def create_parameter_df(
             OBJECTIVE_PRIOR_PARAMETERS: "",
         }
     )
-    df.set_index([PARAMETER_ID], inplace=True)
+    df = df.set_index([PARAMETER_ID])
 
     # For model parameters, set nominal values as defined in the model
     for parameter_id in df.index:

petab/v1/simplify.py

@@ -23,8 +23,7 @@ def remove_nan_measurements(problem: Problem):
     """Drop any measurements that are NaN"""
     problem.measurement_df = problem.measurement_df[
         ~problem.measurement_df[MEASUREMENT].isna()
-    ]
-    problem.measurement_df.reset_index(inplace=True, drop=True)
+    ].reset_index(drop=True)
 
 
 def remove_unused_observables(problem: Problem):
@@ -103,8 +102,7 @@ def condition_parameters_to_parameter_table(problem: Problem):
         }
         for parameter_id, value in replacements.items()
     ]
-    rows = pd.DataFrame(rows)
-    rows.set_index(PARAMETER_ID, inplace=True)
+    rows = pd.DataFrame(rows).set_index(PARAMETER_ID)
 
     if problem.parameter_df is None:
         problem.parameter_df = rows

petab/v1/visualize/data_overview.py

@@ -83,7 +83,10 @@ def get_data_per_observable(measurement_df: pd.DataFrame) -> pd.DataFrame:
     )
 
     # Add row and column sums
-    data_per_observable.loc["SUM", :] = data_per_observable.sum(axis=0).values
+    sum_row = "SUM" if len(index) == 1 else ("SUM", "")
+    data_per_observable.loc[sum_row, :] = data_per_observable.sum(
+        axis=0
+    ).values
     data_per_observable["SUM"] = data_per_observable.sum(axis=1).values
 
     data_per_observable = data_per_observable.astype(int)