fix unk format when k>1

Author: theblackcat102

fastlangid/langid.py

@@ -94,7 +94,12 @@ def predict(self, text, full_clean=False, supplement_threshold=0.9, k=1, prob=Fa
         if isinstance(text, unicode):
             text = self.clean_up(text)
             if len(text) == 0 or only_punctuations(text[:50]):
-                return (UNK_CLS, 1.0) if prob else UNK_CLS
+                output = UNK_CLS
+                if prob:
+                    output = (UNK_CLS, 1.0)
+                if k > 1:
+                    output = [output]
+                return output
             return self._predict_text(text, supplement_threshold=supplement_threshold, k=k, prob=prob, force_second=force_second)
         else:
             batch = [ self.clean_up(i, full_clean=full_clean) for i in text ]

setup.py

@@ -27,7 +27,7 @@
 
 setup(
     name='fastlangid',
-    version='1.0.8',
+    version='1.0.9',
     description='Language detection for news powered by fasttext',
     long_description=readme,
     long_description_content_type="text/markdown",

tests/testcases.py

@@ -1,6 +1,7 @@
 # coding=utf-8
 from __future__ import division, unicode_literals
 from fastlangid.langid import LID
+from fastlangid.utils import UNK_CLS
 from os import path
 import unittest
 
@@ -126,8 +127,18 @@ def test_batches(self):
             ])
         self.assertEqual(lang_codes,  ['zh-hant', 'ru', 'de', 'en','it', 'fr', 'ja', 'pt'])
 
+    def test_unknown(self):
+        lang_code = self.langid.predict('', force_second=True)
+        self.assertEqual(lang_code, UNK_CLS)
 
+        lang_code = self.langid.predict('???｡???｡???｡???｡???｡???｡')
+        self.assertEqual(lang_code, UNK_CLS)
 
+        lang_code = self.langid.predict('???｡???｡???｡???｡???｡???｡', prob=True)
+        self.assertEqual(lang_code, (UNK_CLS, 1.0))
+
+        lang_code = self.langid.predict('???｡???｡???｡???｡???｡???｡', prob=True, k=15)
+        self.assertEqual(lang_code, [(UNK_CLS, 1.0)])
 
 def suite():
     suite = unittest.TestSuite()