Updated implicit and explicit evaler. Saves with the correct metric-name now

Author: Max

lkauto/explicit/explicit_evaler.py

@@ -86,7 +86,7 @@ def __init__(self,
                                                       frac=self.split_frac,
                                                       random_state=self.random_state)
         else:
-            self.train_test_splits = iter([TTSplit(train, validation)])
+            self.train_test_splits = None
 
     def evaluate(self, config_space: ConfigurationSpace) -> float:
         """ evaluates model defined in config_space
@@ -112,54 +112,40 @@ def evaluate(self, config_space: ConfigurationSpace) -> float:
         # get model from configuration space
         model = get_model_from_cs(config_space, feedback='explicit')
 
-        '''
-        # loop over validation folds
-        for fold in range(self.split_folds):
-            if self.validation is None:
-                # get validation split by fold index
-                validation_train = self.train.loc[self.train_test_splits[fold]["train"], :]
-                validation_test = self.train.loc[self.train_test_splits[fold]["validation"], :]
-            else:
-                validation_train = self.train
-                validation_test = self.validation
-
-            # split validation data into X and y
-            x_validation_test = validation_test.copy()
-            y_validation_test = validation_test.copy()
-
-            # process validation split
-            x_validation_test = x_validation_test.drop('rating', inplace=False, axis=1)
-            y_validation_test = y_validation_test[['rating']].iloc[:, 0]
-
+        if self.validation is None:
+            for fold in self.train_test_splits:
+                validation_train = self.train_test_splits.train
+                validation_test = self.train_test_splits.test
 
-            # fit and predict model from configuration
-            model.fit(validation_train)
-            predictions = model.predict(x_validation_test)
-            predictions.index = x_validation_test.index
+                pipeline = predict_pipeline(scorer=model)
+                fit_pipeline = pipeline.clone()
+                fit_pipeline.train(data=validation_train)
 
-            # calculate error_metric and append to numpy array
-            error_metric = np.append(error_metric,
-                                     self.optimization_metric(predictions, y_validation_test, missing='ignore'))
+                recs = predict(fit_pipeline, validation_test)
 
-            validation_data = pd.concat([validation_data, predictions], axis=0)
-            '''
+                run_analysis = RunAnalysis()
+                run_analysis.add_metric(self.optimization_metric)
+                error_results = run_analysis.measure(recs, validation_test)
 
-        for fold in self.train_test_splits:
-            validation_train = fold.train
-            validation_test = fold.test
+                error_metric = np.append(error_metric, error_results)
+                validation_data = pd.concat([validation_data, recs], ignore_index=True)
+        else:
+            for fold in range(self.split_folds):
+                validation_train = self.train
+                validation_test = self.validation
 
-            pipeline = predict_pipeline(scorer=model)
-            fit_pipeline = pipeline.clone()
-            fit_pipeline.train(data=validation_train)
+                pipeline = predict_pipeline(scorer=model)
+                fit_pipeline = pipeline.clone()
+                fit_pipeline.train(data=validation_train)
 
-            recs = predict(fit_pipeline, validation_test.keys())
+                recs = predict(fit_pipeline, validation_test)
 
-            run_analysis = RunAnalysis()
-            run_analysis.add_metric(self.optimization_metric)
-            error_results = run_analysis.measure(recs, validation_test)
+                run_analysis = RunAnalysis()
+                run_analysis.add_metric(self.optimization_metric)
+                error_results = run_analysis.measure(recs, validation_test)
 
-            error_metric = np.append(error_metric, error_results)
-            validation_data = pd.concat([validation_data, recs], ignore_index=True)
+                error_metric = np.append(error_metric, error_results)
+                validation_data = pd.concat([validation_data, recs], ignore_index=True)
 
         # Save validation data for reproducibility and ensembling
         self.top_n_runs = update_top_n_runs(config_space=config_space,

lkauto/implicit/implicit_evaler.py

@@ -6,6 +6,7 @@
 from lenskit import batch
 from lenskit.data import Dataset
 from lenskit.metrics import RunAnalysis
+from lenskit.splitting import TTSplit
 import logging
 
 from lenskit.pipeline import predict_pipeline
@@ -110,33 +111,52 @@ def evaluate(self, config_space: ConfigurationSpace) -> float:
         # get model form configuration space
         model = get_model_from_cs(config_space, feedback='implicit')
 
-        # iterate over validation folds
-        for fold in range(self.split_folds):
-            # get validation split by index
-            if self.validation is None:
-                validation_train = self.val_fold_indices[fold].train
-                validation_test = self.val_fold_indices[fold].test
-            else:
+        if self.validation is None:
+            for fold in self.val_fold_indices:
+                validation_train = fold.train
+                validation_test = fold.test
+
+                pipeline = predict_pipeline(scorer=model)
+                fit_pipeline = pipeline.clone()
+                fit_pipeline.train(validation_train)
+
+                recs = predict(fit_pipeline, validation_test)
+
+                # create rec list analysis
+                rla = RunAnalysis()
+                rla.add_metric(self.optimization_metric)
+
+                # compute scores
+                scores = rla.measure(recs, validation_test)
+
+                # store data
+                validation_data = pd.concat([validation_data, recs.to_df()], axis=0)
+                # the first (index 0) column should contain the means for the metrics (rows)
+                metric_scores = np.append(metric_scores, scores.list_summary()[self.optimization_metric.__name__].iloc[0])
+        else:
+            for fold in range(self.split_folds):
                 validation_train = self.train
                 validation_test = self.validation
 
-            pipeline = predict_pipeline(scorer=model)
-            fit_pipeline = pipeline.clone()
-            fit_pipeline.train(validation_train)
+                pipeline = predict_pipeline(scorer=model)
+                fit_pipeline = pipeline.clone()
+                fit_pipeline.train(validation_train)
+
+                recs = predict(fit_pipeline, validation_test)
 
-            recs = predict(fit_pipeline, validation_test.keys())
+                # create rec list analysis
+                rla = RunAnalysis()
+                rla.add_metric(self.optimization_metric)
 
-            # create rec list analysis
-            rla = RunAnalysis()
-            rla.add_metric(self.optimization_metric)
+                # compute scores
+                scores = rla.measure(recs, validation_test)
 
-            # compute scores
-            scores = rla.measure(recs, validation_test)
+                # store data
+                validation_data = pd.concat([validation_data, recs.to_df()], axis=0)
+                # the first (index 0) column should contain the means for the metrics (rows)
+                metric_scores = np.append(metric_scores,
+                scores.list_summary()[self.optimization_metric.__name__].iloc[0])
 
-            # store data
-            validation_data = pd.concat([validation_data, recs], axis=0)
-            # the first (index 0) column should contain the means for the metrics (rows)
-            metric_scores = np.append(metric_scores, scores.list_summary()[self.optimization_metric.__name].iloc[0])
 
         # save validation data
         self.filer.save_validataion_data(config_space=config_space,

lkauto/lkauto.py

@@ -231,7 +231,7 @@ def get_best_recommender_model(train: Dataset,
                                cs: ConfigurationSpace = None,
                                optimization_metric=NDCG,
                                optimization_strategie: str = 'bayesian',
-                               time_limit_in_sec: int = 2700,
+                               time_limit_in_sec: int = 600,
                                num_evaluations: int = 500,
                                random_state=None,
                                split_folds: int = 1,