Add CMRAds200 adsorption energies benchmarks

docs/source/user_guide/benchmarks/surfaces.rst

@@ -102,6 +102,42 @@ Reference data:
 * Same as input data
 * PBE-D3(BJ), MPRelaxSet settings
 
+CMRAds200
+=========
+
+Summary
+-------
+
+Performance in predicting adsorption energies on several transition metal
+surfaces for a set of 200 adsorbate-surface combinations.
+
+Metrics
+-------
+
+Error in adsorption energy
+
+For each combination of surface, molecule, and surface + molecule, the adsorption
+energy is calculated by taking the difference between the energy of the surface +
+molecule and the sum of individual surface and molecule energies. This is compared to
+the reference adsorption energy, calculated in the same way.
+
+Computational cost
+------------------
+
+Low: tests are likely to take a couple of minutes to run on CPU.
+
+Data availability
+-----------------
+
+Input data:
+
+* 1 P.S. Schmidt, and K.S. Thygesen, “Benchmark database of transition metal surface and adsorption energies from many-body perturbation theory,” J. Phys. Chem. C 122(8), 4381–4390 (2018). https://pubs.acs.org/doi/10.1021/acs.jpcc.7b12258
+
+Reference data:
+
+* Same as input data
+* PBE-D3(BJ), MPRelaxSet settings
+
 Elemental Slab Oxygen Adsorption
 ================================
 

ml_peg/analysis/surfaces/CMRAds200/analyse_CMRAds200.py

@@ -0,0 +1,169 @@
+"""Analyse CMRAds200 benchmark."""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+from ase.io import read, write
+import pytest
+
+from ml_peg.analysis.utils.decorators import build_table, plot_parity
+from ml_peg.analysis.utils.utils import (
+    build_dispersion_name_map,
+    load_metrics_config,
+    mae,
+)
+from ml_peg.app import APP_ROOT
+from ml_peg.calcs import CALCS_ROOT
+from ml_peg.models.get_models import get_model_names
+from ml_peg.models.models import current_models
+
+MODELS = get_model_names(current_models)
+DISPERSION_NAME_MAP = build_dispersion_name_map(MODELS)
+CALC_PATH = CALCS_ROOT / "surfaces" / "CMRAds200" / "outputs"
+OUT_PATH = APP_ROOT / "data" / "surfaces" / "CMRAds200"
+
+METRICS_CONFIG_PATH = Path(__file__).with_name("metrics.yml")
+DEFAULT_THRESHOLDS, DEFAULT_TOOLTIPS, _ = load_metrics_config(METRICS_CONFIG_PATH)
+
+
+def labels() -> list:
+    """
+    Get list of labels.
+
+    Returns
+    -------
+    list
+        List of all energy labels.
+    """
+    structs = read(CALC_PATH / "mol_surface_structs.extxyz", index=":")
+    return [struct.info["sys_formula"] for struct in structs]
+
+
+def system_names() -> list:
+    """
+    Get list of system names.
+
+    Returns
+    -------
+    list
+        List of all system names.
+    """
+    for model_name in MODELS:
+        model_dir = CALC_PATH / model_name
+        if model_dir.exists():
+            structs = read(model_dir / "mol_surface_structs.extxyz", index=":")
+            system_names = [struct.info["sys_formula"] for struct in structs]
+            break
+    return system_names
+
+
+@pytest.fixture
+@plot_parity(
+    filename=OUT_PATH / "figure_adsorption_energies.json",
+    title="Adsorption energies",
+    x_label="Predicted adsorption energy / eV",
+    y_label="Reference adsorption energy / eV",
+    hoverdata={
+        "System": system_names(),
+    },
+)
+def adsorption_energies() -> dict[str, list]:
+    """
+    Get adsorption energies for all systems.
+
+    Returns
+    -------
+    dict[str, list]
+        Dictionary of all reference and predicted adsorption energies.
+    """
+    results = {"ref": []} | {mlip: [] for mlip in MODELS}
+    ref_stored = False
+
+    for model_name in MODELS:
+        model_dir = CALC_PATH / model_name
+        if not model_dir.exists():
+            results[model_name] = []
+            continue
+        mol_surface_list = read(model_dir / "mol_surface_structs.extxyz", index=":")
+        for _mol_surface_idx, mol_surface in enumerate(
+            mol_surface_list
+        ):  # sorted(model_dir.glob("*.xyz")):
+            system_name = mol_surface.info["sys_formula"]
+
+            # Get pre-calculated adsorption energies
+            pred_ads_energy = mol_surface.info["pred_adsorption_energy"]
+            results[model_name].append(pred_ads_energy)
+
+            if not ref_stored:
+                ref_ads_energy = mol_surface.info["PBE_adsorption_energy"]
+                results["ref"].append(ref_ads_energy)
+
+            # Write molecule-surface structure to app data
+            structs_dir = OUT_PATH / model_name
+            structs_dir.mkdir(parents=True, exist_ok=True)
+            write(structs_dir / f"{system_name}.xyz", mol_surface)
+
+        ref_stored = True
+    return results
+
+
+@pytest.fixture
+def cmrads_mae(adsorption_energies) -> dict[str, float]:
+    """
+    Get mean absolute error for adsorption energies.
+
+    Parameters
+    ----------
+    adsorption_energies
+        Dictionary of reference and predicted adsorption energies.
+
+    Returns
+    -------
+    dict[str, float]
+        Dictionary of predicted adsorption energy errors for all models.
+    """
+    results = {}
+    for model_name in MODELS:
+        results[model_name] = mae(
+            adsorption_energies["ref"], adsorption_energies[model_name]
+        )
+    return results
+
+
+@pytest.fixture
+@build_table(
+    filename=OUT_PATH / "cmrads_metrics_table.json",
+    metric_tooltips=DEFAULT_TOOLTIPS,
+    thresholds=DEFAULT_THRESHOLDS,
+    mlip_name_map=DISPERSION_NAME_MAP,
+)
+def metrics(cmrads_mae: dict[str, float]) -> dict[str, dict]:
+    """
+    Get all CMRAds200 metrics.
+
+    Parameters
+    ----------
+    cmrads_mae
+        Mean absolute errors for all models.
+
+    Returns
+    -------
+    dict[str, dict]
+        Metric names and values for all models.
+    """
+    return {
+        "MAE": cmrads_mae,
+    }
+
+
+def test_cmrads200(metrics: dict[str, dict]) -> None:
+    """
+    Run CMRAds200 test.
+
+    Parameters
+    ----------
+    metrics
+        All CMRAds200 metrics.
+    """
+    return

ml_peg/analysis/surfaces/CMRAds200/metrics.yml

@@ -0,0 +1,7 @@
+metrics:
+  MAE:
+    good: 0.10
+    bad: 0.60
+    unit: eV
+    tooltip: Mean Absolute Error
+    level_of_theory: PBE

ml_peg/app/surfaces/CMRAds200/app_CMRAds200.py

@@ -0,0 +1,93 @@
+"""Run CMRAds200 app."""
+
+from __future__ import annotations
+
+from dash import Dash
+from dash.html import Div
+
+from ml_peg.app import APP_ROOT
+from ml_peg.app.base_app import BaseApp
+from ml_peg.app.utils.build_callbacks import (
+    plot_from_table_column,
+    struct_from_scatter,
+)
+from ml_peg.app.utils.load import read_plot
+from ml_peg.models.get_models import get_model_names
+from ml_peg.models.models import current_models
+
+# Get all models
+MODELS = get_model_names(current_models)
+BENCHMARK_NAME = "CMRAds200"
+DOCS_URL = (
+    "https://ddmms.github.io/ml-peg/user_guide/benchmarks/surfaces.html#cmrads200"
+)
+DATA_PATH = APP_ROOT / "data" / "surfaces" / "CMRAds200"
+
+
+class CMRAds200App(BaseApp):
+    """CMRAds200 benchmark app layout and callbacks."""
+
+    def register_callbacks(self) -> None:
+        """Register callbacks to app."""
+        scatter = read_plot(
+            DATA_PATH / "figure_adsorption_energies.json",
+            id=f"{BENCHMARK_NAME}-figure",
+        )
+
+        structs_dir = DATA_PATH / MODELS[0]
+
+        # Assets dir will be parent directory
+        structs = [
+            f"/assets/surfaces/CMRAds200/{MODELS[0]}/{struct_file.stem}.xyz"
+            for struct_file in sorted(structs_dir.glob("*.xyz"))
+        ]
+
+        plot_from_table_column(
+            table_id=self.table_id,
+            plot_id=f"{BENCHMARK_NAME}-figure-placeholder",
+            column_to_plot={"MAE": scatter},
+        )
+
+        struct_from_scatter(
+            scatter_id=f"{BENCHMARK_NAME}-figure",
+            struct_id=f"{BENCHMARK_NAME}-struct-placeholder",
+            structs=structs,
+            mode="traj",
+        )
+
+
+def get_app() -> CMRAds200App:
+    """
+    Get CMRAds200 benchmark app layout and callback registration.
+
+    Returns
+    -------
+    CMRAds200App
+        Benchmark layout and callback registration.
+    """
+    return CMRAds200App(
+        name=BENCHMARK_NAME,
+        description=(
+            "Performance in predicting adsorption energies for 200 "
+            "adsorbate-surface systems."
+        ),
+        docs_url=DOCS_URL,
+        table_path=DATA_PATH / "cmrads_metrics_table.json",
+        extra_components=[
+            Div(id=f"{BENCHMARK_NAME}-figure-placeholder"),
+            Div(id=f"{BENCHMARK_NAME}-struct-placeholder"),
+        ],
+    )
+
+
+if __name__ == "__main__":
+    # Create Dash app
+    full_app = Dash(__name__, assets_folder=DATA_PATH.parent.parent)
+
+    # Construct layout and register callbacks
+    cmrads200_app = get_app()
+    full_app.layout = cmrads200_app.layout
+    cmrads200_app.register_callbacks()
+
+    # Run app
+    full_app.run(port=8056, debug=True)