per sample entropy needs to account for variable length sequences as well

Author: lucidrains

setup.py

@@ -3,7 +3,7 @@
 setup(
   name = 'vector_quantize_pytorch',
   packages = find_packages(),
-  version = '1.12.5',
+  version = '1.12.6',
   license='MIT',
   description = 'Vector Quantization - Pytorch',
   long_description_content_type = 'text/markdown',

vector_quantize_pytorch/lookup_free_quantization.py

@@ -215,13 +215,13 @@ def forward(
 
             prob = (-distance * inv_temperature).softmax(dim = -1)
 
-            per_sample_entropy = entropy(prob).mean()
-
             # account for mask
 
             if exists(mask):
                 prob = prob[mask]
 
+            per_sample_entropy = entropy(prob).mean()
+
             # distribution over all available tokens in the batch
 
             avg_prob = reduce(prob, '... c d -> c d', 'mean')