automatically sync codebook, if distributed is initialized and world size greater than 1

Author: lucidrains

setup.py

@@ -3,7 +3,7 @@
 setup(
   name = 'vector_quantize_pytorch',
   packages = find_packages(),
-  version = '1.7.0',
+  version = '1.7.1',
   license='MIT',
   description = 'Vector Quantization - Pytorch',
   long_description_content_type = 'text/markdown',

vector_quantize_pytorch/vector_quantize_pytorch.py

@@ -710,7 +710,7 @@ def __init__(
         sample_codebook_temp = 1.,
         straight_through = False,
         reinmax = False,  # using reinmax for improved straight-through, assuming straight through helps at all
-        sync_codebook = False,
+        sync_codebook = None,
         sync_affine_param = False,
         ema_update = True,
         learnable_codebook = False,
@@ -760,6 +760,9 @@ def __init__(
             straight_through = straight_through
         )
 
+        if not exists(sync_codebook):
+            sync_codebook = distributed.is_initialized() and distributed.get_world_size() > 1
+
         codebook_kwargs = dict(
             dim = codebook_dim,
             num_codebooks = heads if separate_codebook_per_head else 1,