Merge pull request #16 from jftuga/normalize

Add fraction symbol normalization

Author: jftuga

Makefile

@@ -60,7 +60,7 @@ test-install: clean
 # Production PyPI targets
 prod-publish: clean check-pypirc $(VENV_NAME) build
 	@echo "Are you sure you want to publish to production PyPI? [y/N] " && read ans && [ $${ans:-N} = y ]
-	./$(VENV_NAME)/bin/twine --verbose upload dist/*
+	./$(VENV_NAME)/bin/twine upload --verbose dist/*
 
 prod-install: clean
 	$(PYTHON) -m venv prod-install-venv

deidentification/deidentification_constants.py

@@ -2,7 +2,7 @@
 
 pgmName = "deidentification"
 pgmUrl = "https://github.com/jftuga/deidentification"
-pgmVersion = "1.3.1"
+pgmVersion = "1.3.2"
 
 # the maps the default replacement word for each language
 class DeidentificationLanguages(Enum):

deidentification/normalize_punctuation.py

@@ -8,6 +8,7 @@ def normalize_punctuation(text: str) -> str:
     - Converts ellipsis character to three periods
     - Converts various spaces to regular space
     - Converts bullet points to asterisk
+    - Converts fraction symbols (like ½) to ASCII representations (like 1/2)
     - Preserves but normalizes common symbols (©, ®, ™)
 
     Args:
@@ -63,6 +64,13 @@ def normalize_punctuation(text: str) -> str:
         chr(0x00B7): '*',  # MIDDLE DOT
         chr(0x2219): '*',  # BULLET OPERATOR
 
+        # Fraction characters
+        chr(0x00BD): ' 1/2',  # FRACTION ONE HALF (½)
+        chr(0x00BC): ' 1/4',  # FRACTION ONE QUARTER
+        chr(0x00BE): ' 3/4',  # FRACTION THREE QUARTERS
+        chr(0x2153): ' 1/3',  # FRACTION ONE THIRD
+        chr(0x2154): ' 2/3',  # FRACTION TWO THIRDS
+
         # Normalize common symbols
         chr(0x00A9): '(c)',  # COPYRIGHT SIGN
         chr(0x00AE): '(r)',  # REGISTERED SIGN
@@ -75,3 +83,4 @@ def normalize_punctuation(text: str) -> str:
         normalized_text = normalized_text.replace(unicode_char, ascii_char)
 
     return normalized_text
+