Merge pull request #17 from dataiku/bug/remove-spacymoji

Remove spacymoji and avoid loading stopwords if not required

Author: alexcombessie

code-env/python/spec/requirements.txt

@@ -5,7 +5,7 @@ jieba==0.42.1
 pyvi==0.1
 regex==2020.11.13
 spacy[lookups,th]==2.3.5
-spacymoji==2.0.0
+emoji==1.2.0
 tqdm==4.50.2
 matplotlib==3.3.1
 wordcloud==1.8.0

custom-recipes/nlp-visualization-wordcloud/recipe.py

@@ -17,9 +17,13 @@
 output_partition_path = params["output_partition_path"]
 df = params["df"]
 
+# Instanciate tokenizer
+tokenizer = MultilingualTokenizer(
+    stopwords_folder_path=(params["stopwords_folder_path"] if params["remove_stopwords"] else None)
+)
 # Load wordcloud visualizer
 worcloud_visualizer = WordcloudVisualizer(
-    tokenizer=MultilingualTokenizer(stopwords_folder_path=params["stopwords_folder_path"]),
+    tokenizer=tokenizer,
     text_column=params["text_column"],
     font_folder_path=font_folder_path,
     language=params["language"],

python-lib/spacy_tokenizer.py

@@ -14,7 +14,6 @@
 from spacy.language import Language
 from spacy.tokens import Doc, Token
 from spacy.vocab import Vocab
-from spacymoji import Emoji
 from emoji import UNICODE_EMOJI
 from fastcore.utils import store_attr
 
@@ -176,10 +175,6 @@ def _create_spacy_tokenizer(self, language: AnyStr) -> Language:
                 nlp.tokenizer.prefix_search = spacy.util.compile_prefix_regex(_prefixes).search
         if self.stopwords_folder_path and language in SUPPORTED_LANGUAGES_SPACY:
             self._customize_stopwords(nlp, language)
-        try:
-            nlp.add_pipe(Emoji(nlp), first=True)
-        except (AttributeError, ValueError) as e:
-            logging.warning(f"Spacymoji not available for language '{language}' because of error: '{e}'")
         logging.info(f"Loading tokenizer for language '{language}': done in {perf_counter() - start:.2f} seconds")
         return nlp
 

tests/python/integration/test_wordcloud.py

@@ -22,21 +22,23 @@ def test_wordcloud_multilingual_subcharts(user_dss_clients):
 
 
 def test_wordcloud_multilingual_subcharts_unsupported_languages(user_dss_clients):
-    dss_scenario.run(user_dss_clients, project_key=TEST_PROJECT_KEY, scenario_id="multilingual_subcharts_unsupported_languages")
+    dss_scenario.run(
+        user_dss_clients, project_key=TEST_PROJECT_KEY, scenario_id="multilingual_subcharts_unsupported_languages"
+    )
 
 
 def test_wordcloud_multilingual_subcharts_per_language(user_dss_clients):
     dss_scenario.run(user_dss_clients, project_key=TEST_PROJECT_KEY, scenario_id="subchart_per_language")
 
 
-def test_wordcloud_edge_cases_multilingual(user_clients):
+def test_wordcloud_edge_cases_multilingual(user_dss_clients):
     dss_scenario.run(user_dss_clients, project_key=TEST_PROJECT_KEY, scenario_id="EDGE_CASES")
 
 
 def test_wordcloud_partitioned_folder_file(user_dss_clients):
     dss_scenario.run(user_dss_clients, project_key=TEST_PROJECT_KEY, scenario_id="partitionned_folder_file")
 
-    
+
 def test_wordcloud_partitioned_folder_sql(user_dss_clients):
     dss_scenario.run(user_dss_clients, project_key=TEST_PROJECT_KEY, scenario_id="partitionned_folder_sql")