Release/glm 2.0.0

.gitignore

@@ -3,15 +3,14 @@ tests/allure_report
 __pycache__/
 *.py[cod]
 *$py.class
-
+*.wlock
 # C extensions
 *.so
 
 # Distribution / packaging
 .Python
 build/
 develop-eggs/
-dist/
 downloads/
 eggs/
 .eggs/

CHANGELOG.md

@@ -1,5 +1,15 @@
 # Changelog
 
+## [Version 2.0.0] - New Feature Release - 2025-09
+
+* Visual Webapp to train GLMs and assess their fit
+* Removal of the custom Model View
+* Interactions added to the GLM in the Visual ML
+
+## [Version 1.1.1] - Bugfix Release - 2024-02
+
+* Update the PredictionModelHandler interface
+
 ## [Version 1.1.0] - Upgrade Release - 2024-01
 
 * Switched the GLM library from statsmodels to glum

Makefile

@@ -44,7 +44,7 @@ integration-tests:
 		source env/bin/activate; \
 		pip install --upgrade pip;\
 		pip install --no-cache-dir -r tests/python/integration/requirements.txt; \
-		pytest tests/python/integration --alluredir=tests/allure_report || ret=$$?; exit $$ret \
+		pytest tests/python/integration --exclude-dss-targets="DSS11" --alluredir=tests/allure_report || ret=$$?; exit $$ret \
 	)
 
 tests: unit-tests integration-tests

README.md

@@ -7,9 +7,9 @@ and its accuracy therefore is not able to be guaranteed. The regression spline c
 # Components
 
 The plugin contains the following components:
+- Visual Webapp to train GLM and assess their fit using One-Way Variable Charts, Variable-Level Stats and Lift Charts
 - Generalized Linear Model Regression to run GLM regression inside the visual ML interface
 - Generalized Linear Model Classification to run GLM binary classification inside the visual ML interface
-- Actual vs Expected view inside the Visual Analysis and the Deployed Model windows
 - Regression Spline Prepare step to compute B-Spline basis row by row
 - Regression Spline Recipe to compute B-spline basis in a separate recipe
 

code-env/python/spec/requirements.dev.txt

@@ -0,0 +1,7 @@
+Flask>=0.9
+pandas>=0.23.0
+numpy<1.24
+python-dotenv==0.19.0
+webaiku @ git+https://github.com/dataiku/solutions-contrib.git@main#egg=webaiku&subdirectory=bs-infra
+dataiku-api-client>=11.0.0
+https://design.solutions.dataiku-dss.io/public/packages/dataiku-internal-client.tar.gz

code-env/python/spec/requirements.txt

@@ -1,5 +1,5 @@
 dash==2.14.2
-dash_bootstrap_components==1.5.0
+dash_bootstrap_components
 scipy==1.11.4; python_version=='3.9'
 scipy==1.10.1; python_version=='3.8'
 scipy==1.7.3; python_version=='3.7'
@@ -12,4 +12,7 @@ glum==2.5.0; python_version=='3.7'
 patsy==0.5.4; python_version=='3.9'
 patsy==0.5.3; python_version<'3.9'
 cloudpickle==1.5.0
-urllib3<2
+urllib3<2
+statsmodels
+python-dotenv==0.19.0
+# webaiku @ git+https://github.com/dataiku/solutions-contrib.git@main#egg=webaiku&subdirectory=bs-infra

custom-recipes/regression-splines/recipe.json

@@ -76,4 +76,4 @@
 
     "resourceKeys": []
 
-}
+}

custom-recipes/regression-splines/recipe.py

@@ -49,4 +49,4 @@
 output_dataset_df = regression_splines.run_spline_creation(df)
 
 # Write recipe outputs
-output_dataset.write_with_schema(output_dataset_df)
+output_dataset.write_with_schema(output_dataset_df)

plugin.json

@@ -1,10 +1,10 @@
 {
     "id": "generalized-linear-models",
-    "version": "1.1.0",
+    "version": "2.0.0",
     "meta": {
         "label": "Generalized Linear Models",
         "description": "Train and deploy Generalized Linear Models",
-        "author": "Dataiku (Matthew Galloway, David Behar, Nicolas Vallée)",
+        "author": "Dataiku",
         "icon": "icon-bullseye",
         "tags": ["Machine Learning"],
         "url": "https://www.dataiku.com/product/plugins/glm",

python-lib/backend/api_utils.py

@@ -0,0 +1,161 @@
+import re
+from flask import current_app
+from logging_assist.logging import logger
+from model_cache.model_conformity_checker import ModelConformityChecker
+from .dataiku_api import dataiku_api
+import pandas as pd
+from dku_visual_ml.dku_model_retrival import VisualMLModelRetriver
+from glm_handler.dku_relativites_calculator import RelativitiesCalculator
+from chart_formatters.variable_level_stats import VariableLevelStatsFormatter
+
+def format_models(global_dku_mltask):
+    logger.info("Formatting Models")
+    model_id_pattern = r'\((.*?)\)'
+    mcc = ModelConformityChecker()
+
+    list_ml_id = global_dku_mltask.get_trained_models_ids()
+    project_key = dataiku_api.default_project.project_key
+    ml_task_id = global_dku_mltask.mltask_id
+    analysis_id = global_dku_mltask.analysis_id
+    models = []
+    for ml_id in list_ml_id:
+        model_details = global_dku_mltask.get_trained_model_details(ml_id)
+        is_conform = mcc.check_model_conformity(ml_id)
+        if is_conform:
+            model_name = model_details.get_user_meta()['name']
+            matches = re.findall(model_id_pattern, model_name)
+            date = [v['value'] for v in model_details.get_user_meta()['labels'] if v['key'] == 'model:date'][0]
+            models.append({"id": ml_id, "name": matches[0], "date": date, "project_key": project_key, "ml_task_id": ml_task_id, "analysis_id": analysis_id})
+        else:
+            current_app.logger.info(f"model {ml_id} is not conform")
+    return models
+
+def np_encode(obj):
+    if isinstance(obj, np.int64):
+        return int(obj)
+    return obj
+
+
+def natural_sort_key(s):
+    import re
+    return [int(c) if c.isdigit() else c.lower() for c in re.split(r'(\d+)', str(s))]
+
+def calculate_base_levels(df, exposure_column=None):
+    cols_json = []
+    # Sort the columns using natural sorting
+    sorted_columns = sorted(df.columns, key=natural_sort_key)
+
+    for col in sorted_columns:
+        if col == exposure_column:
+            continue
+
+        # Determine if the column contains numeric or non-numeric data
+        is_numeric = pd.api.types.is_numeric_dtype(df[col])
+
+        if is_numeric:
+            options = sorted([str(val) for val in df[col].unique()], key=float)
+        else:
+            options = sorted([str(val) for val in df[col].unique()], key=natural_sort_key)
+
+        if exposure_column and exposure_column in df.columns:
+            # Exposure-based calculation
+            weighted_counts = df.groupby(col)[exposure_column].sum()
+            base_level = str(weighted_counts.idxmax())
+        else:
+            # Original mode-based calculation
+            base_level = str(df[col].mode().iloc[0])
+
+        cols_json.append({
+            'column': col,
+            'options': options,
+            'baseLevel': base_level,
+            'type': ('numerical' if is_numeric else 'categorical')
+        })
+
+    return cols_json
+
+def get_model_train_set(full_model_id, model_cache, data_handler):
+    model_retriever = VisualMLModelRetriver(full_model_id)
+    relativities_calculator = RelativitiesCalculator(data_handler, model_retriever)
+    train_set = relativities_calculator.train_set
+    return train_set
+
+def get_model_test_set(full_model_id, model_cache, data_handler):
+    model_retriever = VisualMLModelRetriver(full_model_id)
+    relativities_calculator = RelativitiesCalculator(data_handler, model_retriever)
+    test_set = relativities_calculator.test_set
+    return test_set
+
+def get_model_base_values_modalities_types(full_model_id, model_cache, data_handler):
+    creation_args = {"data_handler": data_handler,
+                     "model_cache": model_cache,
+                    "full_model_id": full_model_id}
+    train_set = model_cache.get_or_create_cached_item(full_model_id, 'train_set', get_model_train_set, **creation_args)
+    test_set = model_cache.get_or_create_cached_item(full_model_id, 'test_set', get_model_test_set, **creation_args)
+    model_retriever = VisualMLModelRetriver(full_model_id)
+    relativities_calculator = RelativitiesCalculator(data_handler, model_retriever, train_set, test_set)
+    base_values = relativities_calculator.get_base_values()
+    return {'base_values': base_values, 
+            'modalities': relativities_calculator.modalities, 
+            'types': relativities_calculator.variable_types}
+
+def get_model_relativities(full_model_id, model_cache, data_handler):
+    creation_args = {"data_handler": data_handler,
+                     "model_cache": model_cache,
+                    "full_model_id": full_model_id}
+    train_set = model_cache.get_or_create_cached_item(full_model_id, 'train_set', get_model_train_set, **creation_args)
+    test_set = model_cache.get_or_create_cached_item(full_model_id, 'test_set', get_model_test_set, **creation_args)
+    base_values_modalities_types = model_cache.get_or_create_cached_item(full_model_id, 'base_values_modalities_types', get_model_base_values_modalities_types, **creation_args)
+    base_values = base_values_modalities_types['base_values']
+    modalities = base_values_modalities_types['modalities']
+    variable_types = base_values_modalities_types['types']
+    model_retriever = VisualMLModelRetriver(full_model_id)
+    relativities_calculator = RelativitiesCalculator(data_handler, model_retriever, train_set, test_set, base_values=base_values, modalities=modalities, variable_types=variable_types)
+    relativities = relativities_calculator.get_relativities_df()
+    relativities_dict = relativities_calculator.relativities
+    return {'relativities': relativities, 'relativities_dict': relativities_dict}
+
+def get_model_relativities_interaction(full_model_id, model_cache, data_handler):
+    creation_args = {"data_handler": data_handler,
+                     "model_cache": model_cache,
+                    "full_model_id": full_model_id}
+    train_set = model_cache.get_or_create_cached_item(full_model_id, 'train_set', get_model_train_set, **creation_args)
+    test_set = model_cache.get_or_create_cached_item(full_model_id, 'test_set', get_model_test_set, **creation_args)
+    base_values_modalities_types = model_cache.get_or_create_cached_item(full_model_id, 'base_values_modalities_types', get_model_base_values_modalities_types, **creation_args)
+    base_values = base_values_modalities_types['base_values']
+    modalities = base_values_modalities_types['modalities']
+    variable_types = base_values_modalities_types['types']
+    model_retriever = VisualMLModelRetriver(full_model_id)
+    relativities_calculator = RelativitiesCalculator(data_handler, model_retriever, train_set, test_set, base_values=base_values, modalities=modalities, variable_types=variable_types)
+    relativities_interaction = relativities_calculator.get_relativities_interactions_df()
+    return relativities_interaction
+
+def get_model_variable_level_stats(full_model_id, model_cache, data_handler):
+    creation_args = {"data_handler": data_handler,
+                     "model_cache": model_cache,
+                    "full_model_id": full_model_id}
+    train_set = model_cache.get_or_create_cached_item(full_model_id, 'train_set', get_model_train_set, **creation_args)
+    test_set = model_cache.get_or_create_cached_item(full_model_id, 'test_set', get_model_test_set, **creation_args)
+    relativities = model_cache.get_or_create_cached_item(full_model_id, 'relativities', get_model_relativities, **creation_args)['relativities']
+    relativities_interaction = model_cache.get_or_create_cached_item(full_model_id, 'relativities_interaction', get_model_relativities_interaction, **creation_args)
+    base_values_modalities_types = model_cache.get_or_create_cached_item(full_model_id, 'base_values_modalities_types', get_model_base_values_modalities_types, **creation_args)
+    base_values = base_values_modalities_types['base_values']
+    model_retriever = VisualMLModelRetriver(full_model_id)
+    variable_level_stats = VariableLevelStatsFormatter(model_retriever, data_handler, relativities, relativities_interaction, base_values, train_set, test_set)
+    variable_stats = variable_level_stats.get_variable_level_stats()
+    return variable_stats
+
+def get_model_predicted_base(full_model_id, model_cache, data_handler, variable):
+    creation_args = {"data_handler": data_handler,
+                     "model_cache": model_cache,
+                    "full_model_id": full_model_id}
+    train_set = model_cache.get_or_create_cached_item(full_model_id, 'train_set', get_model_train_set, **creation_args)
+    test_set = model_cache.get_or_create_cached_item(full_model_id, 'test_set', get_model_test_set, **creation_args)
+    base_values_modalities_types = model_cache.get_or_create_cached_item(full_model_id, 'base_values_modalities_types', get_model_base_values_modalities_types, **creation_args)
+    base_values = base_values_modalities_types['base_values']
+    modalities = base_values_modalities_types['modalities']
+    variable_types = base_values_modalities_types['types']
+    model_retriever = VisualMLModelRetriver(full_model_id)
+    relativities_calculator = RelativitiesCalculator(data_handler, model_retriever, train_set, test_set, base_values, modalities, variable_types)
+    predicted_base_variable = relativities_calculator.get_formated_predicted_base_variable(variable)
+    return predicted_base_variable

python-lib/backend/dataiku_api.py

@@ -0,0 +1,44 @@
+import logging
+from typing import Any, Dict
+import dataiku
+from dataiku.customwebapp import get_webapp_config
+import pandas as pd
+from dataiku.customrecipe import get_recipe_config
+import os
+import pwd
+from typing import Optional
+
+class DataikuApi:
+    def __init__(self):
+        self._webapp_config = None
+        self._default_project = None
+        self._default_project_key = None
+        self._client = dataiku.api_client()
+
+    def setup(self, webapp_config: Dict, default_project_key: str):
+        self._webapp_config = webapp_config
+        self._default_project_key = default_project_key
+
+    @property
+    def client(self):
+        if self._client is None:
+            raise Exception("Please set the client before using it.")
+        else:
+            return self._client
+
+    @property
+    def default_project(self):
+        try:
+            return self.client.get_default_project()
+        except Exception as err:
+            if self._default_project_key:
+                return self.client.get_project(self._default_project_key)
+            else:
+                raise Exception("Please define the default project before using it.")
+
+    @property
+    def plugin_code_env(self):
+        plugin = self.client.get_plugin('generalized-linear-models')
+        return plugin.get_settings().get_raw()['codeEnvName']
+
+dataiku_api = DataikuApi()