API add obj|data|solv parameters as columns in result DataFrame (#703)

Co-authored-by: tommoral <thomas.moreau.2010@gmail.com>
benchopt · Sep 25, 2024 · 104e413 · 104e413
1 parent d082c0a
commit 104e413
Show file tree

Hide file tree

Showing 4 changed files with 138 additions and 16 deletions.
diff --git a/benchopt/runner.py b/benchopt/runner.py
@@ -245,15 +245,18 @@ def run_one_to_cvg_cached(**kwargs):
         output.set(rep=rep)
 
         # Get meta
-        meta = dict(
-            objective_name=str(objective),
-            solver_name=str(solver),
-            data_name=str(dataset),
-            idx_rep=rep,
-            sampling_strategy=sampling_strategy.capitalize(),
-            obj_description=obj_description,
-            solver_description=inspect.cleandoc(solver.__doc__ or ""),
-        )
+        meta = {
+            'objective_name': str(objective),
+            'obj_description': obj_description,
+            'solver_name': str(solver),
+            'solver_description': inspect.cleandoc(solver.__doc__ or ""),
+            'data_name': str(dataset),
+            'idx_rep': rep,
+            'sampling_strategy': sampling_strategy.capitalize(),
+            **{f"p_obj_{k}": v for k, v in objective._parameters.items()},
+            **{f"p_solver_{k}": v for k, v in solver._parameters.items()},
+            **{f"p_dataset_{k}": v for k, v in dataset._parameters.items()},
+        }
 
         stopping_criterion = solver._stopping_criterion.get_runner_instance(
             solver=solver,

diff --git a/benchopt/tests/test_runner.py b/benchopt/tests/test_runner.py
@@ -2,6 +2,7 @@
 
 import pytest
 import numpy as np
+import pandas as pd
 
 from benchopt.tests import TEST_DATASET
 from benchopt.tests import TEST_OBJECTIVE
@@ -368,3 +369,87 @@ def test_benchopt_run_script(n_jobs, no_debug_log):
 
     # Make sure the results were saved in a result file
     assert len(out.result_files) == 1, out.output
+
+
+def test_prefix_with_same_parameters():
+    from benchopt import run_benchmark
+
+    solver1 = """from benchopt import BaseSolver
+
+        class Solver(BaseSolver):
+            name = "solver1"
+            sampling_strategy = 'iteration'
+            parameters = dict(seed=[3, 27])
+            def set_objective(self, X, y): pass
+            def run(self, n_iter): pass
+            def get_result(self): return dict(beta=1)
+    """
+
+    # Different name and extra parameter
+    solver2 = (
+        solver1.replace("solver1", "solver2")
+        .replace('seed=[3, 27]', 'seed=[2, 28], type=["s"]')
+    )
+
+    dataset1 = """from benchopt import BaseDataset
+
+        class Dataset(BaseDataset):
+            name = "dataset1"
+            parameters = dict(seed=[3, 27])
+            def get_data(self):
+                return dict(X=0, y=1)
+    """
+
+    # Different name and extra parameter
+    dataset2 = (
+        dataset1.replace("dataset1", "dataset2")
+        .replace('seed=[3, 27]', 'seed=[2, 28], type=["d"]')
+    )
+
+    objective = """from benchopt import BaseObjective
+
+        class Objective(BaseObjective):
+            name = "test_obj"
+            min_benchopt_version = "0.0.0"
+
+            parameters = dict(test_p=[4])
+            def set_data(self, X, y): pass
+            def get_one_result(self): pass
+            def evaluate_result(self, beta): return dict(value=1)
+            def get_objective(self): return dict(X=0, y=0)
+    """
+
+    with temp_benchmark(solvers=[solver1, solver2],
+                        datasets=[dataset1, dataset2],
+                        objective=objective
+                        ) as benchmark:
+        run_benchmark(
+            str(benchmark.benchmark_dir),
+            solver_names=["solver1", "solver2"],
+            dataset_names=["dataset1", "dataset2"],
+            max_runs=1, n_repetitions=1, n_jobs=1, plot_result=False
+        )
+
+        df = pd.read_parquet(benchmark.get_result_file())
+
+        assert "p_solver_seed" in df.columns
+        assert "p_solver_type" in df.columns
+        assert "p_dataset_seed" in df.columns
+        assert "p_dataset_type" in df.columns
+        assert "p_obj_test_p" in df.columns
+
+        assert df.query("p_solver_seed.isna()").shape[0] == 0
+        no_type = df.query("p_solver_type.isna()")['solver_name'].unique()
+        assert all('solver1' in s for s in no_type)
+
+        assert df.query("p_dataset_seed.isna()").shape[0] == 0
+        no_type = df.query("p_dataset_type.isna()")['data_name'].unique()
+        assert all('dataset1' in s for s in no_type)
+
+        assert df.query("p_obj_test_p.isna()").shape[0] == 0
+
+        # No mixing
+        assert "d" not in df['p_solver_type'].unique()
+        assert "s" in df['p_solver_type'].unique()
+        assert "s" not in df['p_dataset_type'].unique()
+        assert "d" in df['p_dataset_type'].unique()
diff --git a/doc/benchmark_workflow/manage_benchmark_results.rst b/doc/benchmark_workflow/manage_benchmark_results.rst
@@ -3,18 +3,47 @@
 Manage benchmark results
 ========================
 
+.. _benchmark_results:
+
+Description of the benchmark results DataFrame
+-----------------------------------------------
+
+
+Once the benchmark is run, the results are stored as a ``pd.DataFrame`` in a
+``.parquet`` file, located in a directory ``./outputs`` in the benchmark
+directory, with a ``.parquet`` file.
+By default, the name of the file include the date and time of the run,
+as ``benchopt_run_<date>_<time>.parquet`` but a custom name can be given using
+the :option:`--output` option of ``benchopt run``.
+The DataFrame contains the following columns:
+
+- ``objective_name|solver_name|data_name``: the names of the different benchopt
+  components used for this line.
+- ``obj_description|solver_description``: A more verbose description of the
+  objective and solver, displayed in the HTML page.
+- ``p_obj_p``: the value of the objective's parameter ``p``.
+- ``p_solver_p``: the value of the solver's parameter ``p``.
+- ``p_dateset_p``: the value of the dataset's parameter ``p``.
+- ``time``: the time taken to run the solver until this point of the performance curve.
+- ``stop_val``: the number of iterations or the tolerance reached by the solver.
+- ``idx_rep``: If multiple repetitions are run for each solver with ``--n-rep``,
+  this column contains the repetition number.
+- ``sampling_strategy``: The sampling strategy used to generate the performance
+  curve of this solver. This allow to adapt the plot in the HTML depending on
+  each solver.
+- ``objective_k``: The value associated to the key ``k`` in the
+  ``Objective.evaluate_result`` dictionary.
+
+The remaining columns are informations about the system used to run the
+benchmark, with keys ``{'env_OMP_NUM_THREADS', 'platform', 'platform-architecture', 'platform-release', 'platform-version', 'system-cpus', 'system-processor', 'system-ram (GB)', 'version-cuda', 'version-numpy', 'version-scipy', 'benchmark-git-tag'}``.
+
+
 .. _collect_results:
 
 Collect benchmark results
 -------------------------
 
-Once the benchmark is run, the results are stored in a directory
-``./results`` in the benchmark directory, with a ``.parquet`` file.
-By default, the name of the file include the date and time of the run,
-but a custom name can be given using the :option:`--output` option of
-``benchopt run``.
-
-This result file is produced only once the full benchmark has been run.
+The result file is produced only once the full benchmark has been run.
 When the benchmark is run in parallel, the results that have already been
 computed can be collected using the :option:`--collect` option with
 ``benchopt run``. Adding this option with the same command line will

diff --git a/doc/whats_new.rst b/doc/whats_new.rst
@@ -26,6 +26,11 @@ API
   with ``::`` instead of ``:``. This allow specifying URL channels.
   By `Thomas Moreau`_ (:gh:`758`)
 
+- Add ``Objective``, ``Solver`` and ``Dataset`` parameters as columns in the
+  result DataFrame. The parameters' names are respectively prefixed with
+  ``p_obj_|p_solver_|p_dataset_`` to avoid collapse between the different
+  components. By `Melvine Nargeot`_  and `Thomas Moreau`_ (:gh:`703`).
+
 FIX
 ---