Merge branch 'update_lkauto' of https://github.com/ISG-Siegen/lenskit-auto into update_lkauto

Author: Catnatsuki

lkauto/explicit/explicit_evaler.py

@@ -1,7 +1,16 @@
+import logging
+
 import numpy as np
 import pandas as pd
+
+from typing import Iterator, Union
+from lenskit.data import Dataset, ItemListCollection
+from lenskit.pipeline import predict_pipeline, topn_pipeline
+from lenskit.batch import recommend
+from lenskit.metrics import RunAnalysis
+from lenskit.splitting import TTSplit
 from ConfigSpace import ConfigurationSpace
-import logging
+from sklearn.model_selection import train_test_split
 
 from lkauto.utils.filer import Filer
 from lkauto.utils.get_model_from_cs import get_model_from_cs
@@ -47,10 +56,10 @@ class ExplicitEvaler:
     """
 
     def __init__(self,
-                 train: pd.DataFrame,
+                 train: Dataset,
                  optimization_metric,
                  filer: Filer,
-                 validation=None,
+                 validation: ItemListCollection = None,
                  random_state=42,
                  split_folds: int = 1,
                  split_strategie: str = 'user_based',
@@ -72,13 +81,13 @@ def __init__(self,
         self.ensemble_size = ensemble_size
         self.top_n_runs = pd.DataFrame(columns=['run_id', 'model', 'error'])
         if self.validation is None:
-            self.val_fold_indices = validation_split(data=self.train,
-                                                     strategy=self.split_strategie,
-                                                     num_folds=self.split_folds,
-                                                     frac=self.split_frac,
-                                                     random_state=self.random_state)
+            self.train_test_splits = validation_split(data=self.train,
+                                                      strategy=self.split_strategie,
+                                                      num_folds=self.split_folds,
+                                                      frac=self.split_frac,
+                                                      random_state=self.random_state)
         else:
-            self.val_fold_indices = None
+            self.train_test_splits = iter([TTSplit(train, validation)])
 
     def evaluate(self, config_space: ConfigurationSpace) -> float:
         """ evaluates model defined in config_space
@@ -104,34 +113,54 @@ def evaluate(self, config_space: ConfigurationSpace) -> float:
         # get model from configuration space
         model = get_model_from_cs(config_space, feedback='explicit')
 
+        '''
         # loop over validation folds
         for fold in range(self.split_folds):
             if self.validation is None:
                 # get validation split by fold index
-                validation_train = self.train.loc[self.val_fold_indices[fold]["train"], :]
-                validation_test = self.train.loc[self.val_fold_indices[fold]["validation"], :]
+                validation_train = self.train.loc[self.train_test_splits[fold]["train"], :]
+                validation_test = self.train.loc[self.train_test_splits[fold]["validation"], :]
             else:
                 validation_train = self.train
                 validation_test = self.validation
 
             # split validation data into X and y
-            X_validation_test = validation_test.copy()
+            x_validation_test = validation_test.copy()
             y_validation_test = validation_test.copy()
 
             # process validation split
-            X_validation_test = X_validation_test.drop('rating', inplace=False, axis=1)
+            x_validation_test = x_validation_test.drop('rating', inplace=False, axis=1)
             y_validation_test = y_validation_test[['rating']].iloc[:, 0]
 
+
             # fit and predict model from configuration
             model.fit(validation_train)
-            predictions = model.predict(X_validation_test)
-            predictions.index = X_validation_test.index
+            predictions = model.predict(x_validation_test)
+            predictions.index = x_validation_test.index
 
             # calculate error_metric and append to numpy array
             error_metric = np.append(error_metric,
                                      self.optimization_metric(predictions, y_validation_test, missing='ignore'))
 
             validation_data = pd.concat([validation_data, predictions], axis=0)
+            '''
+
+        for fold in self.train_test_splits:
+            validation_train = fold.train
+            validation_test = fold.test
+
+            pipeline = predict_pipeline(scorer=model)
+            fit_pipeline = pipeline.clone()
+            fit_pipeline.train(data=validation_train)
+
+            recs = recommend(fit_pipeline, validation_test.keys())
+
+            run_analysis = RunAnalysis()
+            run_analysis.add_metric(self.optimization_metric)
+            error_results = run_analysis.measure(recs, validation_test)
+
+            error_metric = np.append(error_metric, error_results)
+            validation_data = pd.concat([validation_data, recs], ignore_index=True)
 
         # Save validation data for reproducibility and ensembling
         self.top_n_runs = update_top_n_runs(config_space=config_space,

lkauto/lkauto.py

@@ -19,12 +19,13 @@
 from lenskit.metrics.predict import RMSE
 from lenskit.metrics import NDCG
 from lenskit.pipeline import Component
+from lenskit.data import Dataset, ItemListCollection
 
 from typing import Tuple
 
 
-def get_best_prediction_model(train: pd.DataFrame,
-                              validation: pd.DataFrame = None,
+def get_best_prediction_model(train: Dataset,
+                              validation: ItemListCollection = None,
                               cs: ConfigurationSpace = None,
                               optimization_metric=RMSE,
                               optimization_strategie: str = 'bayesian',
@@ -166,7 +167,7 @@ def get_best_prediction_model(train: pd.DataFrame,
                     drop_na_values=drop_na_values,
                     drop_duplicates=drop_duplicates)
 
-    # decide which optimization strategie to use
+    # decide which optimization strategy to use
     if optimization_strategie == 'bayesian':
         incumbent, top_n_runs = bayesian_optimization(train=train,
                                                       cs=cs,

lkauto/optimization_strategies/bayesian_optimization.py

@@ -9,6 +9,8 @@
 from smac.intensifier import Intensifier
 from smac.scenario import Scenario
 
+from lenskit.data import Dataset, ItemListCollection
+
 from lkauto.explicit.explicit_evaler import ExplicitEvaler
 from lkauto.implicit.implicit_evaler import ImplicitEvaler
 from lkauto.utils.filer import Filer
@@ -19,9 +21,9 @@
 import logging
 
 
-def bayesian_optimization(train: pd.DataFrame,
+def bayesian_optimization(train: Dataset,
                           user_feedback: str,
-                          validation: pd.DataFrame = None,
+                          validation: ItemListCollection = None,
                           cs: ConfigurationSpace = None,
                           optimization_metric=None,
                           time_limit_in_sec: int = 2700,
@@ -118,7 +120,7 @@ def bayesian_optimization(train: pd.DataFrame,
     if cs is None:
         logger.debug('initializing default ConfigurationSpace')
         cs = get_default_configuration_space(data=train,
-                                             val_fold_indices=evaler.val_fold_indices,
+                                             val_fold_indices=evaler.train_test_splits,
                                              validation=validation,
                                              feedback='explicit',
                                              random_state=random_state)

lkauto/optimization_strategies/random_search.py

@@ -4,6 +4,8 @@
 
 from ConfigSpace import ConfigurationSpace, Configuration
 
+from lenskit.data import Dataset, ItemListCollection
+
 from lkauto.explicit.explicit_evaler import ExplicitEvaler
 from lkauto.implicit.implicit_evaler import ImplicitEvaler
 from lkauto.utils.get_default_configurations import get_default_configurations
@@ -15,11 +17,11 @@
 
 
 def random_search(cs: ConfigurationSpace,
-                  train: pd.DataFrame,
+                  train: Dataset,
                   user_feedback: str,
                   optimization_metric,
                   filer: Filer,
-                  validation: pd.DataFrame = None,
+                  validation: ItemListCollection = None,
                   time_limit_in_sec: int = 3600,
                   num_evaluations: int = None,
                   split_folds: int = 1,
@@ -117,7 +119,7 @@ def random_search(cs: ConfigurationSpace,
     if cs is None:
         logger.debug('initializing default ConfigurationSpace')
         cs = get_default_configuration_space(data=train,
-                                             val_fold_indices=evaler.val_fold_indices,
+                                             val_fold_indices=evaler.train_test_splits,
                                              validation=validation,
                                              feedback='explicit',
                                              random_state=random_state)

lkauto/preprocessing/pruning.py

@@ -1,59 +1,63 @@
-import pandas as pd
+#import pandas as pd
+from lenskit.data import Dataset,from_interactions_df
 
-
-def min_ratings_per_user(df: pd.DataFrame, num_ratings: int, count_duplicates: bool = False):
+def min_ratings_per_user(dataset: Dataset, num_ratings: int, count_duplicates: bool = False):
     """Prune users with less than num_ratings ratings
 
     Parameters
     ----------
-    df: pd.DataFrame
-        Dataframe with columns "user", "item", "rating"
+    dataset: Dataset
+        LensKit Dataset object containing user-item interactions with ratings
     num_ratings: int
         Minimum number of ratings per user
     count_duplicates: bool = False
         If True, all ratings are counted, otherwise only unique ratings are counted
 
     Returns
     -------
-    pd.DataFrame
-        Dataframe with columns "user", "item", "rating"
+    Dataset
+        Filtered Dataset with only users meeting the minimum rating threshold
+        the Dataset will contain the columns "user_id", "item_id", "rating"
     """
-    # get all relevant user_ids
-    uids = (
-        df['user']
-        if count_duplicates
-        else df.drop_duplicates(['user', 'item'])['user']
-    )
-    cnt_items_per_user = uids.value_counts()
-    users_of_interest = list(cnt_items_per_user[cnt_items_per_user >= num_ratings].index)
+    # get the user statistics from the dataset
+    user_stats = dataset.user_stats()
+    if count_duplicates: 
+        valid_users = user_stats[user_stats['count'] >= num_ratings].index # count: total number of ratings (including duplicates)
+    else:
+        valid_users = user_stats[user_stats['item_count'] >= num_ratings].index # item_count: number of unique items rated
+    # convert the interaction table to a pandas DataFrame and filter by valid users
+    users_of_interest = dataset.interaction_table(format='pandas', original_ids=True)
+    users_of_interest = users_of_interest[users_of_interest['user_id'].isin(valid_users)]
+    return from_interactions_df(users_of_interest)
+
 
-    return df[df['user'].isin(users_of_interest)]
 
 
-def max_ratings_per_user(df: pd.DataFrame, num_ratings: int, count_duplicates: bool = False):
+def max_ratings_per_user(dataset: Dataset, num_ratings: int, count_duplicates: bool = False):
     """Prune users with more than num_ratings ratings
 
     Parameters
     ----------
-    df: pd.DataFrame
-        Dataframe with columns "user", "item", "rating"
+    dataset: Dataset
+        LensKit Dataset object containing user-item interactions with ratings
     num_ratings: int
-        Minimum number of ratings per user
+        Maximum number of ratings per user
     count_duplicates: bool = False
         If True, all ratings are counted, otherwise only unique ratings are counted
 
     Returns
     -------
-    pd.DataFrame
-        Dataframe with columns "user", "item", "rating"
+    Dataset
+        Filtered Dataset with only users meeting the minimum rating threshold
+        the Dataset will contain the columns "user_id", "item_id", "rating"
     """
-    # get all relevant user_ids
-    uids = (
-        df['user']
-        if count_duplicates
-        else df.drop_duplicates(['user', 'item'])['user']
-    )
-    cnt_items_per_user = uids.value_counts()
-    users_of_interest = list(cnt_items_per_user[cnt_items_per_user <= num_ratings].index)
-
-    return df[df['user'].isin(users_of_interest)]
+
+    user_stats = dataset.user_stats()
+    if count_duplicates: 
+        valid_users = user_stats[user_stats['count'] <= num_ratings].index # count: total number of ratings (including duplicates)
+    else:
+        valid_users = user_stats[user_stats['item_count'] <= num_ratings].index # item_count: number of unique items rated
+    # convert the interaction table to a pandas DataFrame and filter by valid users
+    users_of_interest = dataset.interaction_table(format='pandas', original_ids=True)
+    users_of_interest = users_of_interest[users_of_interest['user_id'].isin(valid_users)]
+    return from_interactions_df(users_of_interest)

lkauto/utils/get_default_configuration_space.py

@@ -45,25 +45,6 @@ def get_default_configuration_space(data: Union[Dataset, Iterator[TTSplit]],
         raise ValueError("Unknown feedback type: {}".format(feedback))
 
     # get minimum number of items and users for the given train split
-
-    """
-    num_items = 0
-    num_users = 0
-    if validation is None:
-        val_fold_indices = val_fold_indices
-        for fold in range(len(val_fold_indices)):
-            tmp = data.loc[val_fold_indices[fold]["train"], :]
-            if tmp['item'].nunique() < num_items or num_items == 0:
-                num_items = tmp['item'].nunique()
-            if tmp['user'].nunique() < num_users or num_users == 0:
-                num_users = tmp['user'].nunique()
-    else:
-        if data['item'].nunique() < num_items or num_items == 0:
-            num_items = data['item'].nunique()
-        if data['user'].nunique() < num_users or num_users == 0:
-            num_users = data['user'].nunique()
-            """
-
     num_items = 0
     num_users = 0
 

lkauto/utils/validation_split.py

@@ -137,12 +137,6 @@ def __row_based_k_fold_validation_split(data: Dataset, num_folds: int, random_st
         Lenskit Dataset with the data to be split.
     """
 
-    # generate the indices of the train and validation split for the given data
-    for i, splits in enumerate(crossfold_records(data, partitions=num_folds, rng_spec=random_state)):
-        fold_indices[i]['train'] = np.append(fold_indices[i]["train"], splits[0].index)
-        fold_indices[i]['validation'] = np.append(fold_indices[i]["validation"], splits[1].index)
-    return fold_indices
-
     splits = crossfold_records(data=data, partitions=num_folds, rng=random_state)
     return splits