Convert to uv

.python-version

@@ -1 +1 @@
-3.12.3
+3.11

lsd/design/tf.yaml

@@ -1,3 +1,72 @@
 #  ╭──────────────────────────────────────────────────────────╮
-#  │ Transformer Sample Configuration                         │
+#  │ Transformer Multiverse Configuration                     │
 #  ╰──────────────────────────────────────────────────────────╯
+
+data_choices:
+  Transformer:
+    CNN:
+      name: cnn_dailymail
+      version: ['3.0.0']
+      split: ['train']
+      host: ['huggingface']
+      num_samples: [1000, 5000]
+
+    ArXiv:
+      name: arxiv
+      version: ['1.0.0']
+      split: ['train']
+      host: ['huggingface']
+      num_samples: [1000, 5000]
+
+    LocalText:
+      name: local_corpus
+      path: ['./data/text_corpus.txt']
+      host: ['local']
+      num_samples: [1000]
+
+model_choices:
+  Transformer:
+    DistilBERT:
+      name: ['distilbert-base-uncased']
+      module: ['lsd.generate.transformers.models.huggingface']
+      max_length: [512]
+
+    MiniLM:
+      name: ['sentence-transformers/all-MiniLM-L6-v2']
+      module: ['lsd.generate.transformers.models.sbert']
+      max_length: [384]
+
+    MPNET:
+      name: ['sentence-transformers/all-mpnet-base-v2']
+      module: ['lsd.generate.transformers.models.sbert']
+      max_length: [384]
+
+    Mistral:
+      name: ['mistralai/Mistral-7B-Instruct-v0.1']
+      module: ['lsd.generate.transformers.models.huggingface']
+      max_length: [512, 1024]
+
+    RoBERTa:
+      name: ['roberta-base']
+      module: ['lsd.generate.transformers.models.huggingface']
+      max_length: [512]
+
+implementation_choices:
+  Transformer:
+    Tokenizer:
+      name: ['standard_tokenizer']
+      aggregation: ['mean', 'cls']
+      batch_size: [16, 32]
+      device: ['cpu', 'cuda']
+
+    SentenceEmbedder:
+      name: ['sentence_embedder']
+      aggregation: ['mean']
+      batch_size: [32, 64]
+      device: ['cpu', 'cuda']
+      normalize_embeddings: [true, false]
+
+logging:
+  wandb_logging: false
+  out_file: true
+  dev: false

lsd/generate/transformers/models/sbert.py

@@ -40,10 +40,14 @@ def load_model(self):
         """
         Loads the SentenceTransformer model specified in the config.
 
-        The model is loaded from the `model_name` key in the config dictionary.
+        The model is loaded from the `name` key in the config dictionary.
         """
         model_name = self.config.get("name", "all-MiniLM-L6-v2")
-        self.model = ST(model_name)
+
+        try:
+            self.model = ST(model_name)
+        except Exception as e:
+            raise RuntimeError(f"Failed to load SentenceTransformer model '{model_name}': {e}")
 
     def process_text(self, text: str):
         """

lsd/generate/transformers/tf.py

@@ -27,89 +27,95 @@ def train(self) -> None:
     def generate(self):
         """
         Generate latent representations using the loaded transformer model.
-        
+
         This method processes the loaded dataset through the transformer model
         to generate latent embeddings that are saved for further analysis.
         """
-        if not hasattr(self, 'dataset') or self.dataset is None:
+        if not hasattr(self, "dataset") or self.dataset is None:
             raise ValueError("Dataset not loaded. Call load_data() first.")
-
-        if not hasattr(self, 'model') or self.model is None:
-            raise ValueError("Model not initialized. Call initialize_model() first.")
-
+
+        if not hasattr(self, "model") or self.model is None:
+            raise ValueError(
+                "Model not initialized. Call initialize_model() first."
+            )
+
         # Extract text from dataset based on dataset type
         texts = self._extract_texts_from_dataset()
-        
+
         print(f"Generating embeddings for {len(texts)} text samples...")
-        
+
         # Generate embeddings using the model
         model_instance = self.model(self.tf_cfg)
         embeddings = model_instance.embed(texts)
-        
+
         # Convert to numpy array if it's a tensor
-        if hasattr(embeddings, 'numpy'):
+        if hasattr(embeddings, "numpy"):
             embeddings = embeddings.numpy()
-        elif hasattr(embeddings, 'detach'):
+        elif hasattr(embeddings, "detach"):
             embeddings = embeddings.detach().numpy()
-        
+
         return embeddings
-    
+
     def _extract_texts_from_dataset(self):
         """
         Extract text content from the loaded dataset.
-        
+
         Returns
         -------
         List[str]
             List of text strings extracted from the dataset.
         """
         texts = []
-        
+
         # Handle different dataset structures
-        if hasattr(self.dataset, 'to_iterable_dataset'):
+        if hasattr(self.dataset, "to_iterable_dataset"):
             # HuggingFace dataset object
             for item in self.dataset:
-                if 'article' in item:
-                    texts.append(item['article'])
-                elif 'text' in item:
-                    texts.append(item['text'])
-                elif 'sentence' in item:
-                    texts.append(item['sentence'])
+                if "article" in item:
+                    texts.append(item["article"])
+                elif "text" in item:
+                    texts.append(item["text"])
+                elif "sentence" in item:
+                    texts.append(item["sentence"])
                 else:
                     # Try to find any string field
                     for key, value in item.items():
-                        if isinstance(value, str) and len(value) > 10:  # Reasonable text length
+                        if (
+                            isinstance(value, str) and len(value) > 10
+                        ):  # Reasonable text length
                             texts.append(value)
                             break
         elif isinstance(self.dataset, list):
             # Check if it's a list of dictionaries or a list of strings
             if self.dataset and isinstance(self.dataset[0], dict):
                 # List of dictionaries (like HuggingFace format)
                 for item in self.dataset:
-                    if 'article' in item:
-                        texts.append(item['article'])
-                    elif 'text' in item:
-                        texts.append(item['text'])
-                    elif 'sentence' in item:
-                        texts.append(item['sentence'])
+                    if "article" in item:
+                        texts.append(item["article"])
+                    elif "text" in item:
+                        texts.append(item["text"])
+                    elif "sentence" in item:
+                        texts.append(item["sentence"])
                     else:
                         # Try to find any string field
                         for key, value in item.items():
-                            if isinstance(value, str) and len(value) > 10:  # Reasonable text length
+                            if (
+                                isinstance(value, str) and len(value) > 10
+                            ):  # Reasonable text length
                                 texts.append(value)
                                 break
             else:
                 # List of text strings
                 texts = self.dataset
-        elif hasattr(self.dataset, '__iter__'):
+        elif hasattr(self.dataset, "__iter__"):
             # Any iterable
             texts = list(self.dataset)
         else:
             raise ValueError(f"Unsupported dataset type: {type(self.dataset)}")
-            
+
         if not texts:
             raise ValueError("No text content found in dataset")
-            
+
         return texts
 
     def initialize_model(self, tf_cfg: ConfigType) -> None:
@@ -143,11 +149,11 @@ def _load_remote(self, tf_cfg: ConfigType):
 
         # Load the dataset from HuggingFace
         dataset = load_dataset(dataset_name, version, split=split)
-        
+
         # Limit samples if specified
         if num_samples is not None:
             dataset = dataset.select(range(min(num_samples, len(dataset))))
-            
+
         return dataset
 
     def _load_local_data(self, tf_cfg: ConfigType):