Hindi TN Future Implementations 2.0. - Fraction, Measure and Time

Jenkinsfile

@@ -27,7 +27,7 @@ pipeline {
     HY_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/03-12-24-0'
     MR_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/03-12-24-1'
     JA_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/10-17-24-1'
-    HI_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/04-22-25-0'
+    HI_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/08-06-25-0'
     DEFAULT_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/06-08-23-0'
   }
   stages {

nemo_text_processing/text_normalization/hi/data/measure/quarterly_units.tsv

@@ -4,9 +4,11 @@ h	घंटे
 min	मिनट
 doz	दर्जन
 yr	साल
-yr	वर्ष
 hp	हॉर्सपॉवर
 d	दिन
 month	महीना
 months	महीने
-हफ़्ते हफ़्ते
+हफ़्ते
+सप्ताह
+सदियां
+सदियों

nemo_text_processing/text_normalization/hi/data/measure/unit.tsv

@@ -134,7 +134,6 @@ KHz	किलोहर्ट्ज़
 N	न्यूटन
 dB	डेसीबल
 yr	साल
-yr	वर्ष
 hp	हॉर्सपॉवर
 d	दिन
 month	महीना

nemo_text_processing/text_normalization/hi/data/whitelist/paune_mappings.tsv

@@ -0,0 +1,100 @@
+०	एक
+१	दो
+२	तीन
+३	चार
+४	पाँच
+५	छह
+६	सात
+७	आठ
+८	नौ
+९	दस
+१०	ग्यारह
+११	बारह
+१२	तेरह
+१३	चौदह
+१४	पंद्रह
+१५	सोलह
+१६	सत्रह
+१७	अठारह
+१८	उन्नीस
+१९	बीस
+२०	इक्कीस
+२१	बाईस
+२२	तेईस
+२३	चौबीस
+२४	पच्चीस
+२५	छब्बीस
+२६	सत्ताईस
+२७	अट्ठाईस
+२८	उनतीस
+२९	तीस
+३०	इकतीस
+३१	बत्तीस
+३२	तैंतीस
+३३	चौंतीस
+३४	पैंतीस
+३५	छत्तीस
+३६	सैंतीस
+३७	अड़तीस
+३८	उनतालीस
+३९	चालीस
+४०	इकतालीस
+४१	बयालीस
+४२	तैंतालीस
+४३	चौवालीस
+४४	पैंतालीस
+४५	छियालीस
+४६	सैंतालीस
+४७	अड़तालीस
+४८	उनचास
+४९	पचास
+५०	इक्यावन
+५१	बावन
+५२	तिरेपन
+५३	चौवन
+५४	पचपन
+५५	छप्पन
+५६	सत्तावन
+५७	अट्ठावन
+५८	उनसठ
+५९	साठ
+६०	इकसठ
+६१	बासठ
+६२	तिरेसठ
+६३	चौंसठ
+६४	पैंसठ
+६५	छियासठ
+६६	सड़सठ
+६७	अड़सठ
+६८	उनहत्तर
+६९	सत्तर
+७०	इकहत्तर
+७१	बहत्तर
+७२	तिहत्तर
+७३	चौहत्तर
+७४	पचहत्तर
+७५	छिहत्तर
+७६	सतहत्तर
+७७	अठहत्तर
+७८	उनासी
+७९	अस्सी
+८०	इक्यासी
+८१	बयासी
+८२	तिरासी
+८३	चौरासी
+८४	पचासी
+८५	छियासी
+८६	सत्तासी
+८७	अट्ठासी
+८८	नवासी
+८९	नब्बे
+९०	इक्यानबे
+९१	बानबे
+९२	तिरानबे
+९३	चौरानबे
+९४	पंचानबे
+९५	छियानबे
+९६	सत्तानबे
+९७	अट्ठानबे
+९८	निन्यानबे
+९९	एक सौ

nemo_text_processing/text_normalization/hi/taggers/cardinal.py

@@ -21,12 +21,12 @@
 
 class CardinalFst(GraphFst):
     """
-      Finite state transducer for classifying cardinals, e.g.
-          -२३ -> cardinal { negative: "true"  integer: "तेइस" } }
-    s
-      Args:
-          deterministic: if True will provide a single transduction option,
-              for False multiple transduction are generated (used for audio-based normalization)
+    Finite state transducer for classifying cardinals, e.g.
+        -२३ -> cardinal { negative: "true"  integer: "तेइस" }
+
+    Args:
+        deterministic: if True will provide a single transduction option,
+            for False multiple transduction are generated (used for audio-based normalization)
     """
 
     def __init__(self, deterministic: bool = True, lm: bool = False):
@@ -37,6 +37,10 @@ def __init__(self, deterministic: bool = True, lm: bool = False):
         teens_ties = pynini.string_file(get_abs_path("data/numbers/teens_and_ties.tsv"))
         teens_and_ties = pynutil.add_weight(teens_ties, -0.1)
 
+        self.digit = digit
+        self.zero = zero
+        self.teens_and_ties = teens_and_ties
+
         def create_graph_suffix(digit_graph, suffix, zeros_counts):
             zero = pynutil.add_weight(pynutil.delete("०"), -0.1)
             if zeros_counts == 0:

nemo_text_processing/text_normalization/hi/taggers/decimal.py

@@ -58,9 +58,7 @@ class DecimalFst(GraphFst):
     def __init__(self, cardinal: GraphFst, deterministic: bool = True):
         super().__init__(name="decimal", kind="classify", deterministic=deterministic)
 
-        graph_digit = pynini.string_file(get_abs_path("data/numbers/digit.tsv"))
-        graph_digit |= pynini.string_file(get_abs_path("data/numbers/zero.tsv"))
-
+        graph_digit = cardinal.digit | cardinal.zero
         cardinal_graph = cardinal.final_graph
 
         self.graph = graph_digit + pynini.closure(insert_space + graph_digit).optimize()

nemo_text_processing/text_normalization/hi/taggers/fraction.py

@@ -16,6 +16,7 @@
 from pynini.lib import pynutil
 
 from nemo_text_processing.text_normalization.hi.graph_utils import GraphFst
+from nemo_text_processing.text_normalization.hi.utils import get_abs_path
 
 
 class FractionFst(GraphFst):
@@ -47,13 +48,43 @@ def __init__(self, cardinal, deterministic: bool = True):
         )
         self.denominator = pynutil.insert("denominator: \"") + cardinal_graph + pynutil.insert("\"")
 
-        self.graph = (
+        dedh_dhai_graph = pynini.string_map([("१ १/२", "डेढ़"), ("२ १/२", "ढाई")])
+
+        savva_numbers = cardinal_graph + pynini.cross(" १/४", "")
+        savva_graph = pynutil.insert("सवा ") + savva_numbers
+
+        sadhe_numbers = cardinal_graph + pynini.cross(" १/२", "")
+        sadhe_graph = pynutil.insert("साढ़े ") + sadhe_numbers
+
+        paune = pynini.string_file(get_abs_path("data/whitelist/paune_mappings.tsv"))
+        paune_numbers = paune + pynini.cross(" ३/४", "")
+        paune_graph = pynutil.insert("पौने ") + paune_numbers
+
+        graph_dedh_dhai = pynutil.insert("morphosyntactic_features: \"") + dedh_dhai_graph + pynutil.insert("\" ")
+
+        graph_savva = pynutil.insert("morphosyntactic_features: \"") + savva_graph + pynutil.insert("\" ")
+
+        graph_sadhe = pynutil.insert("morphosyntactic_features: \"") + sadhe_graph + pynutil.insert("\" ")
+
+        graph_paune = pynutil.insert("morphosyntactic_features: \"") + paune_graph + pynutil.insert("\" ")
+
+        final_graph = (
             self.optional_graph_negative
             + pynini.closure(self.integer + pynini.accep(" "), 0, 1)
             + self.numerator
             + self.denominator
         )
 
+        weighted_graph = (
+            final_graph
+            | pynutil.add_weight(graph_dedh_dhai, -0.2)
+            | pynutil.add_weight(graph_savva, -0.1)
+            | pynutil.add_weight(graph_sadhe, -0.1)
+            | pynutil.add_weight(graph_paune, -0.2)
+        )
+
+        self.graph = weighted_graph
+
         graph = self.graph
-        final_graph = self.add_tokens(graph)
-        self.fst = final_graph.optimize()
+        graph = self.add_tokens(graph)
+        self.fst = graph.optimize()

nemo_text_processing/text_normalization/hi/taggers/measure.py

@@ -41,8 +41,9 @@ def __init__(self, cardinal: GraphFst, decimal: GraphFst):
         super().__init__(name="measure", kind="classify")
 
         cardinal_graph = (
-            digit
-            | teens_and_ties
+            cardinal.zero
+            | cardinal.digit
+            | cardinal.teens_and_ties
             | cardinal.graph_hundreds
             | cardinal.graph_thousands
             | cardinal.graph_ten_thousands
@@ -52,6 +53,7 @@ def __init__(self, cardinal: GraphFst, decimal: GraphFst):
         point = pynutil.delete(".")
         decimal_integers = pynutil.insert("integer_part: \"") + cardinal_graph + pynutil.insert("\"")
         decimal_graph = decimal_integers + point + insert_space + decimal.graph_fractional
+
         unit_graph = pynini.string_file(get_abs_path("data/measure/unit.tsv"))
         quarterly_units_graph = pynini.string_file(get_abs_path("data/measure/quarterly_units.tsv"))
 
@@ -93,10 +95,50 @@ def __init__(self, cardinal: GraphFst, decimal: GraphFst):
             + unit
         )
 
-        graph_quarter = (
+        dedh_dhai = pynini.string_map([("१.५", "डेढ़"), ("२.५", "ढाई")])
+        dedh_dhai_graph = pynutil.insert("integer: \"") + dedh_dhai + pynutil.insert("\"")
+
+        savva_numbers = cardinal_graph + pynini.cross(".२५", "")
+        savva_graph = pynutil.insert("integer: \"सवा ") + savva_numbers + pynutil.insert("\"")
+
+        sadhe_numbers = cardinal_graph + pynini.cross(".५", "")
+        sadhe_graph = pynutil.insert("integer: \"साढ़े ") + sadhe_numbers + pynutil.insert("\"")
+
+        paune = pynini.string_file(get_abs_path("data/whitelist/paune_mappings.tsv"))
+        paune_numbers = paune + pynini.cross(".७५", "")
+        paune_graph = pynutil.insert("integer: \"पौने ") + paune_numbers + pynutil.insert("\"")
+
+        graph_dedh_dhai = (
+            pynutil.insert("cardinal { ")
+            + optional_graph_negative
+            + dedh_dhai_graph
+            + pynutil.insert(" }")
+            + delete_space
+            + units
+        )
+
+        graph_savva = (
+            pynutil.insert("cardinal { ")
+            + optional_graph_negative
+            + savva_graph
+            + pynutil.insert(" }")
+            + delete_space
+            + units
+        )
+
+        graph_sadhe = (
+            pynutil.insert("cardinal { ")
+            + optional_graph_negative
+            + sadhe_graph
+            + pynutil.insert(" }")
+            + delete_space
+            + units
+        )
+
+        graph_paune = (
             pynutil.insert("cardinal { ")
             + optional_graph_negative
-            + quarter_graph
+            + paune_graph
             + pynutil.insert(" }")
             + delete_space
             + units
@@ -135,9 +177,12 @@ def __init__(self, cardinal: GraphFst, decimal: GraphFst):
 
         graph = (
             pynutil.add_weight(graph_decimal, 0.01)
-            | pynutil.add_weight(graph_quarter, 0.005)
             | pynutil.add_weight(graph_cardinal, 0.01)
             | pynutil.add_weight(graph_exceptions, 0.01)
+            | pynutil.add_weight(graph_dedh_dhai, 0.001)
+            | pynutil.add_weight(graph_savva, 0.005)
+            | pynutil.add_weight(graph_sadhe, 0.005)
+            | pynutil.add_weight(graph_paune, -0.2)
         )
         self.graph = graph.optimize()
 

nemo_text_processing/text_normalization/hi/taggers/time.py

@@ -36,10 +36,11 @@ class TimeFst(GraphFst):
             for False multiple transduction are generated (used for audio-based normalization)
     """
 
-    def __init__(self):
+    def __init__(self, cardinal: GraphFst):
         super().__init__(name="time", kind="classify")
 
         delete_colon = pynutil.delete(":")
+        cardinal_graph = cardinal.digit | cardinal.teens_and_ties
 
         self.hours = pynutil.insert("hours: \"") + hours_graph + pynutil.insert("\" ")
         self.minutes = pynutil.insert("minutes: \"") + minutes_graph + pynutil.insert("\" ")
@@ -56,7 +57,35 @@ def __init__(self):
         # hour
         graph_h = self.hours + delete_colon + pynutil.delete("००")
 
-        final_graph = graph_hms | graph_hm | graph_h
+        dedh_dhai_graph = pynini.string_map([("१:३०", "डेढ़"), ("२:३०", "ढाई")])
+
+        savva_numbers = cardinal_graph + pynini.cross(":१५", "")
+        savva_graph = pynutil.insert("सवा ") + savva_numbers
+
+        sadhe_numbers = cardinal_graph + pynini.cross(":३०", "")
+        sadhe_graph = pynutil.insert("साढ़े ") + sadhe_numbers
+
+        paune = pynini.string_file(get_abs_path("data/whitelist/paune_mappings.tsv"))
+        paune_numbers = paune + pynini.cross(":४५", "")
+        paune_graph = pynutil.insert("पौने ") + paune_numbers
+
+        graph_dedh_dhai = pynutil.insert("morphosyntactic_features: \"") + dedh_dhai_graph + pynutil.insert("\" ")
+
+        graph_savva = pynutil.insert("morphosyntactic_features: \"") + savva_graph + pynutil.insert("\" ")
+
+        graph_sadhe = pynutil.insert("morphosyntactic_features: \"") + sadhe_graph + pynutil.insert("\" ")
+
+        graph_paune = pynutil.insert("morphosyntactic_features: \"") + paune_graph + pynutil.insert("\" ")
+
+        final_graph = (
+            graph_hms
+            | pynutil.add_weight(graph_hm, 0.01)
+            | pynutil.add_weight(graph_h, 0.01)
+            | pynutil.add_weight(graph_dedh_dhai, 0.001)
+            | pynutil.add_weight(graph_savva, 0.005)
+            | pynutil.add_weight(graph_sadhe, 0.005)
+            | pynutil.add_weight(graph_paune, 0.001)
+        )
 
         final_graph = self.add_tokens(final_graph)
         self.fst = final_graph.optimize()

nemo_text_processing/text_normalization/hi/taggers/tokenize_and_classify.py

@@ -98,7 +98,7 @@ def __init__(
             logging.debug(f"date: {time.time() - start_time: .2f}s -- {date_graph.num_states()} nodes")
 
             start_time = time.time()
-            timefst = TimeFst()
+            timefst = TimeFst(cardinal=cardinal)
             time_graph = timefst.fst
             logging.debug(f"time: {time.time() - start_time: .2f}s -- {time_graph.num_states()} nodes")
 

nemo_text_processing/text_normalization/hi/taggers/word.py

@@ -43,7 +43,6 @@ def __init__(self, punctuation: PunctuationFst, deterministic: bool = True):
             *[chr(i) for i in range(ord("ऀ"), ord("ः") + 1)],  # Hindi vowels and consonants
             *[chr(i) for i in range(ord("अ"), ord("ह") + 1)],  # More Hindi characters
             *[chr(i) for i in range(ord("ा"), ord("्") + 1)],  # Hindi diacritics
-            *[chr(i) for i in range(ord("०"), ord("९") + 1)],  # Hindi digits
         ).optimize()
 
         # Include punctuation in the graph