ENH Added evaluation

docs/model_card.rst

@@ -58,6 +58,10 @@ template which includes the following slots for free text sections:
   out, in case of questions about the model or the model card.
 - ``"citation_bibtex"``: Bibtex style citations for the model or resources used
   to train the model.
+- ``"eval_methods"``: Details about evaluation process of the model.
+- ``"eval_results"``: Evaluation results that are later parsed as a table by
+  :class:`skops.card.Card`.
+
 
 The template also contains the following sections that are automatically
 generated by ``skops``.
@@ -66,7 +70,7 @@ generated by ``skops``.
 - ``"model_plot"``: A diagram of the model, most relevant in case the model is
   a complex scikit-learn :class:`~sklearn.pipeline.Pipeline`.
 
-You can also add other sections to the model  card by passing arbitrary
+You can also add other sections to the model card by passing arbitrary
 keywords to :meth:`.Card.add`.
 
 To see how you can use the API in ``skops`` to create a model card, please

examples/plot_model_card.py

@@ -19,7 +19,12 @@
 from sklearn.datasets import load_breast_cancer
 from sklearn.ensemble import HistGradientBoostingClassifier
 from sklearn.experimental import enable_halving_search_cv  # noqa
-from sklearn.metrics import ConfusionMatrixDisplay, confusion_matrix
+from sklearn.metrics import (
+    ConfusionMatrixDisplay,
+    accuracy_score,
+    confusion_matrix,
+    f1_score,
+)
 from sklearn.model_selection import HalvingGridSearchCV, train_test_split
 
 from skops import card, hub_utils
@@ -115,6 +120,14 @@
     model_description=model_description,
 )
 y_pred = model.predict(X_test)
+model_card.add(
+    eval_method=(
+        "The model is evaluated using test split, on accuracy and F1-score with macro"
+        " average."
+    )
+)
+model_card.add_metrics(accuracy=accuracy_score(y_test, y_pred))
+model_card.add_metrics(**{"f1 score": f1_score(y_test, y_pred, average="micro")})
 cm = confusion_matrix(y_test, y_pred, labels=model.classes_)
 disp = ConfusionMatrixDisplay(confusion_matrix=cm, display_labels=model.classes_)
 disp.plot()

skops/card/_model_card.py

@@ -78,10 +78,10 @@ def metadata_from_config(config_path: Union[str, Path]) -> CardData:
 class Card:
     """Model card class that will be used to generate model card.
 
-    This class can be used to write information and plots to model card and
-    save it. This class by default generates an interactive plot of the model
-    and a table of hyperparameters. The slots to be filled are defined in the
-    markdown template.
+    This class can be used to write information and plots to model card and save
+    it. This class by default generates an interactive plot of the model and a
+    table of hyperparameters. The slots to be filled are defined in the markdown
+    template.
 
     Parameters
     ----------
@@ -118,9 +118,14 @@ class Card:
 
     Examples
     --------
+    >>> from sklearn.metrics import (
+    ...     ConfusionMatrixDisplay,
+    ...     confusion_matrix,
+    ...     accuracy_score,
+    ...     f1_score
+    ... )
     >>> import tempfile
     >>> from pathlib import Path
-    >>> from sklearn.metrics import ConfusionMatrixDisplay, confusion_matrix
     >>> from sklearn.datasets import load_iris
     >>> from sklearn.linear_model import LogisticRegression
     >>> from skops import card
@@ -129,6 +134,15 @@ class Card:
     >>> model_card = card.Card(model)
     >>> model_card.metadata.license = "mit"
     >>> y_pred = model.predict(X)
+    >>> model_card.add_metrics(**{
+    ...     "accuracy": accuracy_score(y, y_pred),
+    ...     "f1 score": f1_score(y, y_pred, average="micro"),
+    ... })
+    Card(
+      model=LogisticRegression(random_state=0),
+      metadata.license=mit,
+    )
+
     >>> cm = confusion_matrix(y, y_pred,labels=model.classes_)
     >>> disp = ConfusionMatrixDisplay(
     ...     confusion_matrix=cm,
@@ -157,6 +171,7 @@ def __init__(
         self.model = model
         self._hyperparameter_table = self._extract_estimator_config()
         # the spaces in the pipeline breaks markdown, so we replace them
+        self._eval_results = {}  # type: ignore
         if model_diagram is True:
             self._model_plot: str | None = re.sub(
                 r"\n\s+", "", str(estimator_html_repr(model))
@@ -205,6 +220,23 @@ def add_plot(self, **kwargs: str) -> "Card":
             self._figure_paths[plot_name] = plot_path
         return self
 
+    def add_metrics(self, **kwargs: str) -> "Card":
+        """Add metric values to the model card.
+
+        Parameters
+        ----------
+        **kwargs : dict
+            A dictionary of the form `{metric name: metric value}`.
+
+        Returns
+        -------
+        self : object
+            Card object.
+        """
+        for metric, value in kwargs.items():
+            self._eval_results[metric] = value
+        return self
+
     def save(self, path: str | Path) -> None:
         """Save the model card.
 
@@ -223,8 +255,14 @@ def save(self, path: str | Path) -> None:
         """
         root = skops.__path__
 
-        template_sections = copy.deepcopy(self._template_sections)
+        # add evaluation results
 
+        template_sections = copy.deepcopy(self._template_sections)
+        template_sections["eval_results"] = tabulate(
+            list(self._eval_results.items()),
+            headers=["Metric", "Value"],
+            tablefmt="github",
+        )
         # if template path is not given, use default
         if template_sections.get("template_path") is None:
             template_sections["template_path"] = str(

skops/card/default_template.md

@@ -29,6 +29,14 @@ The model plot is below.
 
 {{ model_plot }}
 
+## Evaluation Results
+
+You can find the details about evaluation process and the evaluation results.
+
+{{ eval_methods }}
+
+{{ eval_results | default("[More Information Needed]", true)}}
+
 # How to Get Started with the Model
 
 Use the code below to get started with the model.

skops/card/tests/test_card.py

@@ -114,6 +114,15 @@ def test_metadata_keys(destination_path, model_card):
         assert "tags: dummy" in f.read()
 
 
+def test_add_metrics(destination_path, model_card):
+    model_card.add_metrics(**{"acc": 0.1})
+    model_card.add_metrics(f1=0.1)
+    model_card.save(Path(destination_path) / "README.md")
+    with open(Path(destination_path) / "README.md", "r") as f:
+        card = f.read()
+        assert ("acc" in card) and ("f1" in card) and ("0.1" in card)
+
+
 def test_metadata_from_config_tabular_data(destination_path):
     # test if widget data is correctly set in the README
     X, y = load_iris(return_X_y=True, as_frame=True)