Skip to content

Commit c3e1feb

Browse files
isabelizimmisabelizimm
authored
Merge pull request #108 from rstudio/refactor
adding docs, small refactoring
2 parents f7315be + 0754ad8 commit c3e1feb

File tree

6 files changed

+802
-113
lines changed

6 files changed

+802
-113
lines changed

‎docs/source/index.rst

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,7 @@ Version
3939
~VetiverModel
4040
~vetiver_pin_write
4141
~vetiver_create_ptype
42+
~model_card
4243

4344
Deploy
4445
==================

‎vetiver/data/chicago.csv

Lines changed: 699 additions & 101 deletions
Large diffs are not rendered by default.

‎vetiver/monitor.py

Lines changed: 73 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ def compute_metrics(
1111
metric_set: list,
1212
truth: str,
1313
estimate: str,
14+
**kw,
1415
) -> pd.DataFrame:
1516
"""
1617
Compute metrics for given time period
@@ -32,6 +33,8 @@ def compute_metrics(
3233
3334
Example
3435
-------
36+
>>> from datetime import timedelta
37+
>>> import pandas as pd
3538
>>> from sklearn.metrics import mean_squared_error, mean_absolute_error
3639
>>> df = pd.DataFrame(
3740
... {
@@ -62,7 +65,9 @@ def compute_metrics(
6265
"index": i.index[0],
6366
"n": len(i),
6467
"metric": m.__qualname__,
65-
"estimate": m(y_pred=i[truth], y_true=i[estimate]),
68+
# TODO: non-y_pred and y_true metrics
69+
# TODO: multioutput metrics
70+
"estimate": m(y_pred=i[estimate], y_true=i[truth], **kw),
6671
}
6772
]
6873

@@ -105,11 +110,50 @@ def pin_metrics(
105110
The column in df_metrics containing the aggregated dates or datetimes.
106111
Note that this defaults to a column named "index".
107112
overwrite: bool
108-
If TRUE (the default), overwrite any metrics for
109-
dates that exist both in the existing pin and
110-
new metrics with the new values. If FALSE, error
111-
when the new metrics contain overlapping dates with
113+
If True, overwrite any metrics for dates that exist both
114+
in the existing pin and new metrics with the new values.
115+
If False, error when the new metrics contain overlapping dates with
112116
the existing pin.
117+
118+
Example
119+
-------
120+
>>> import pins
121+
>>> import vetiver
122+
>>> df = pd.DataFrame(
123+
... {'index': {0: pd.Timestamp('2021-01-01 00:00:00'),
124+
... 1: pd.Timestamp('2021-01-01 00:00:00'),
125+
... 2: pd.Timestamp('2021-01-02 00:00:00'),
126+
... 3: pd.Timestamp('2021-01-02 00:00:00')},
127+
... 'n': {0: 1, 1: 1, 2: 1, 3: 1},
128+
... 'metric': {0: 'mean_squared_error',
129+
... 1: 'mean_absolute_error',
130+
... 2: 'mean_squared_error',
131+
... 3: 'mean_absolute_error'},
132+
... 'estimate': {0: 4.0, 1: 2.0, 2: 1.0, 3: 1.0}}
133+
... )
134+
>>> board = pins.board_temp()
135+
136+
>>> board.pin_write(df, "metrics", type = "csv") # doctest: +SKIP
137+
>>> df = pd.DataFrame(
138+
... {'index': {0: pd.Timestamp('2021-01-02 00:00:00'),
139+
... 1: pd.Timestamp('2021-01-02 00:00:00'),
140+
... 2: pd.Timestamp('2021-01-03 00:00:00'),
141+
... 3: pd.Timestamp('2021-01-03 00:00:00')},
142+
... 'n': {0: 1, 1: 1, 2: 1, 3: 1},
143+
... 'metric': {0: 'mean_squared_error',
144+
... 1: 'mean_absolute_error',
145+
... 2: 'mean_squared_error',
146+
... 3: 'mean_absolute_error'},
147+
... 'estimate': {0: 4.0, 1: 6.0, 2: 2.0, 3: 1.0}}
148+
... )
149+
>>> vetiver.pin_metrics( # doctest: +SKIP
150+
... board=board,
151+
... df_metrics=df2,
152+
... metrics_pin_name="metrics",
153+
... index_name="index",
154+
... overwrite=True)
155+
156+
113157
"""
114158

115159
old_metrics_raw = board.pin_read(metrics_pin_name)
@@ -170,6 +214,30 @@ def plot_metrics(
170214
Column in `df_metrics` containing metric name
171215
n: str
172216
Column in `df_metrics` containing number of observations
217+
218+
Example
219+
-------
220+
>>> import vetiver
221+
>>> import pandas as pd
222+
>>> df = pd.DataFrame(
223+
... {'index': {0: pd.Timestamp('2021-01-01 00:00:00'),
224+
... 1: pd.Timestamp('2021-01-01 00:00:00'),
225+
... 2: pd.Timestamp('2021-01-02 00:00:00'),
226+
... 3: pd.Timestamp('2021-01-02 00:00:00')},
227+
... 'n': {0: 1, 1: 1, 2: 1, 3: 1},
228+
... 'metric': {0: 'mean_squared_error',
229+
... 1: 'mean_absolute_error',
230+
... 2: 'mean_squared_error',
231+
... 3: 'mean_absolute_error'},
232+
... 'estimate': {0: 4.0, 1: 2.0, 2: 1.0, 3: 1.0}}
233+
... )
234+
>>> plot = vetiver.plot_metrics(
235+
... df_metrics = df,
236+
... date = "index",
237+
... estimate = "estimate",
238+
... metric = "metric",
239+
... n = "n")
240+
>>> plot.show() # doctest: +SKIP
173241
"""
174242

175243
fig = px.line(

‎vetiver/pin_read_write.py

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,9 @@
1-
import warnings
21
from .vetiver_model import VetiverModel
2+
from .utils import inform
3+
import warnings
4+
import logging
5+
6+
_log = logging.getLogger(__name__)
37

48

59
class ModelCard(UserWarning):
@@ -30,8 +34,12 @@ def vetiver_pin_write(board, model: VetiverModel, versioned: bool = True):
3034
if not board.allow_pickle_read:
3135
raise NotImplementedError # must be pickle-able
3236

33-
warnings.simplefilter("once", ModelCard)
34-
warnings.warn(ModelCard().message)
37+
inform(
38+
_log,
39+
"Model Cards provide a framework for transparent, responsible "
40+
"reporting. \n Use the vetiver `.qmd` Quarto template as a place to start, \n "
41+
"with vetiver.model_card()",
42+
)
3543

3644
board.pin_write(
3745
model.model,

‎vetiver/tests/test_rsconnect.py

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -69,16 +69,17 @@ def test_board_pin_write(rsc_short):
6969

7070
@pytest.mark.xfail
7171
def test_deploy(rsc_short):
72-
# TODO: test in Dockerfile
7372
v = vetiver.VetiverModel(
7473
model=model, ptype_data=X_df, model_name="susan/model", versioned=None
7574
)
7675

7776
vetiver.vetiver_pin_write(board=rsc_short, model=v)
78-
77+
connect_server = RSConnectServer(
78+
url=RSC_SERVER_URL, api_key=server_from_key("susan")
79+
)
7980
vetiver.deploy_rsconnect(
80-
connect_server=server_from_key("susan"), board=rsc_short, pin_name="susan/model"
81+
connect_server=connect_server, board=rsc_short, pin_name="susan/model"
8182
)
82-
response = vetiver.predict(RSC_SERVER_URL + "/predict/", json=X_df)
83+
response = vetiver.predict(RSC_SERVER_URL + "/predict", json=X_df)
8384
assert response.status_code == 200, response.text
8485
assert response.json() == {"prediction": [44.47, 44.47]}, response.json()

‎vetiver/utils.py

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
11
import nest_asyncio
22
import warnings
3+
import sys
4+
from types import SimpleNamespace
35

46
no_notebook = False
57
try:
@@ -18,3 +20,14 @@ def _jupyter_nb():
1820
nest_asyncio.apply()
1921
else:
2022
return False
23+
24+
25+
modelcard_options = SimpleNamespace(quiet=False)
26+
27+
28+
def inform(log, msg):
29+
if log is not None:
30+
log.info(msg)
31+
32+
if not modelcard_options.quiet:
33+
print(msg, file=sys.stderr)

0 commit comments

Comments
 (0)