Allow ignoring the temperature parameter by setting it to zero

Author: batzner

README.md

@@ -34,7 +34,7 @@ with tf.Session() as session:
 
 This should output something like:
 
-    The Wtestath  t s   ien  es a   a     ug   dm     ooi  el e       a   s i  n    k s  a    u  ta    o e
+    The  ee e e e e e e e e e e e e e e e e e e e e e e e e e e e e e e e e e e e e e e e e e e e e e e e e 
 
 ## Command Line Usage
 

examples/readme/basic_usage.py

@@ -7,7 +7,7 @@
                    neurons_per_layer=100, num_layers=3, num_timesteps=15)
 
     # Train it
-    model.train(session, max_epochs=10, max_steps=500, print_logs=True)
+    model.train(session, max_epochs=10, max_steps=500)
 
     # Let it generate a text
     generated = model.sample(session, "The ", num_steps=100)

requirements.txt

@@ -1,4 +1,4 @@
 numpy==1.13.1
-tensorflow==1.1.0
+# tensorflow==1.1.0
 nltk==3.2.4
 python-dateutil==2.6.1

setup.py

@@ -31,7 +31,6 @@
     ],
     install_requires=[
         "numpy==1.13.1",
-        "tensorflow==1.1.0",
         "nltk==3.2.4",
         "python-dateutil==2.6.1",
     ],

tensorlm/model.py

@@ -206,7 +206,7 @@ def evaluate(self, session, dataset):
         self.will_resume_training(session)
         return total_loss / step_count
 
-    def sample_ids(self, session, prime_ids, num_steps=100, temperature=0.5):
+    def sample_ids(self, session, prime_ids, num_steps=100, temperature=0):
         """Let the model generate a sequence based on a preceding string.
 
         This method primes the model with the given sequence of token ids. Then, it feeds the model
@@ -219,7 +219,8 @@ def sample_ids(self, session, prime_ids, num_steps=100, temperature=0.5):
             num_steps (int): The number of tokens generated by the model.
             temperature (float): Degree of randomness during sampling. The logits returned by the
                 model will be divided by the temperature value before calculating the softmax.
-                The temperature will be clipped to 0.01 if it is below this bound.
+                If the temperature is below 0.01, the model will choose the token with the highest
+                predicted probability at each step instead of sampling from the distribution.
 
         Returns:
             list[int]: The generated sequence ids.
@@ -249,13 +250,13 @@ def sample_ids(self, session, prime_ids, num_steps=100, temperature=0.5):
         self.will_resume_training(session)
         return outputs
 
-    def sample_text(self, session, vocabulary, prime, num_steps=100, temperature=0.5):
+    def sample_text(self, session, vocabulary, prime, num_steps=100, temperature=0):
         """Let the model generate a sequence based on a preceding string.
 
         This method tokenizes the prime string and feeds the tokens to the model. Then, it feeds the
-        model its own output (disgusting, I know) token by token and thus lets it generate /
-        complete the text. For char level, this will result in 100 generated characters, for word
-        level 100 generated tokens (words / punctuation / whitespace).
+        model its own output token by token and thus lets it generate / complete the text. For char
+        level, this will result in 100 generated characters, for word level 100 generated tokens
+        (words / punctuation / whitespace).
 
         Args:
             session (tf.Session): The TF session to run the operations in.
@@ -265,7 +266,8 @@ def sample_text(self, session, vocabulary, prime, num_steps=100, temperature=0.5
             num_steps (int): The number of tokens generated by the model.
             temperature (float): Degree of randomness during sampling. The logits returned by the
                 model will be divided by the temperature value before calculating the softmax.
-                The temperature will be clipped to 0.01 if it is below this bound.
+                If the temperature is below 0.01, the model will choose the token with the highest
+                predicted probability at each step instead of sampling from the distribution.
 
         Returns:
             str: The generated text.
@@ -428,7 +430,7 @@ def will_resume_training(self, session):
         # Re-enable dropout and return to the previous training state
         session.run([self._output_keep_var.assign(self.output_keep_prob), self._unfreeze_state_op])
 
-    def _sample_step(self, session, inputs, update_state=True, temperature=0.5):
+    def _sample_step(self, session, inputs, update_state=True, temperature=0):
         """Feeds batch inputs to the model and returns the batch output ids.
 
         Args:
@@ -443,7 +445,8 @@ def _sample_step(self, session, inputs, update_state=True, temperature=0.5):
                 be frozen before and unfrozen after this function call.
             temperature (float): Degree of randomness during sampling. The logits returned by the
                 model will be divided by the temperature value before calculating the softmax.
-                The temperature will be clipped to 0.01 if it is below this bound.
+                If the temperature is below 0.01, the model will choose the token with the highest
+                predicted probability at each step instead of sampling from the distribution.
 
         Returns:
             np.ndarray: A batch of outputs with the same shape and data type as the inputs
@@ -455,14 +458,16 @@ def _sample_step(self, session, inputs, update_state=True, temperature=0.5):
 
         # Get the output
         logits, _ = session.run(runs, feed_dict=feed_dict)
-        temperature = max(temperature, 0.01)
-
-        # Sample from the output using the probability distribution in logits
-        ids = range(logits.shape[2])
-        result = np.zeros(logits.shape[0:2], dtype=np.uint8)
-        for batch in range(logits.shape[0]):
-            for step in range(logits.shape[1]):
-                probs = np.exp(logits[batch, step] / temperature)
-                probs /= np.sum(probs)
-                result[batch, step] = np.random.choice(ids, p=probs)
+
+        if temperature < 0.01:
+            result = np.argmax(logits, axis=2)
+        else:
+            result = np.zeros(logits.shape[0:2], dtype=np.uint8)
+            # Sample from the output using the probability distribution in logits
+            ids = range(logits.shape[2])
+            for batch in range(logits.shape[0]):
+                for step in range(logits.shape[1]):
+                    probs = np.exp(logits[batch, step] / temperature)
+                    probs /= np.sum(probs)
+                    result[batch, step] = np.random.choice(ids, p=probs)
         return result

tensorlm/wrappers.py

@@ -174,7 +174,7 @@ def evaluate(self, tf_session, text_path):
         loss = self.tf_model.evaluate(tf_session, dataset)
         return loss
 
-    def sample(self, tf_session, prime, num_steps, temperature=0.5):
+    def sample(self, tf_session, prime, num_steps, temperature=0):
         """Let the model generate text after being primed with some text.
 
         Args:
@@ -185,7 +185,8 @@ def sample(self, tf_session, prime, num_steps, temperature=0.5):
                 in num_steps words / numbers / punctuation marks / whitespace characters
             temperature (float): Degree of randomness during sampling. The logits returned by the
                 model will be divided by the temperature value before calculating the softmax.
-                The temperature will be clipped to 0.01 if it is below this bound.
+                If the temperature is below 0.01, the model will choose the token with the highest
+                predicted probability at each step instead of sampling from the distribution.
 
         Returns:
             str: The generated sequence.