add unicode normalization transform

ayoisaiah · Jan 1, 2025 · dd14dbd · dd14dbd
1 parent 69054a4
commit dd14dbd
Show file tree

Hide file tree

Showing 3 changed files with 20 additions and 1 deletion.
diff --git a/replace/replace_test/variables_test.go b/replace/replace_test/variables_test.go
@@ -85,6 +85,18 @@ func TestVariables(t *testing.T) {
 				"$1 - {<$2>.up}{ext.ti}",
 			},
 		},
+		{
+			Name: "normalize unicode NFKC",
+			Changes: file.Changes{
+				{
+					Source: "𝐇𝐞𝐥𝐥𝐨 Ｗｏｒｌｄ Ⓗⓘ ᵂᵒʳˡᵈ 𝟙𝟚𝟛 ℍ𝕚 ℕ𝕦𝕞𝕓𝕖𝕣𝕤 １２３ ①②③ ⒈⒉⒊",
+				},
+			},
+			Want: []string{
+				"Hello World Hi World 123 Hi Numbers 123 123 1.2.3.",
+			},
+			Args: []string{"-f", ".*", "-r", "{.norm}"},
+		},
 		{
 			Name: "remove diacritics",
 			Changes: file.Changes{

diff --git a/replace/variables/variable_regex.go b/replace/variables/variable_regex.go
@@ -55,7 +55,7 @@ func init() {
 	tokenString := strings.Join(tokens, "|")
 
 	transformTokens = fmt.Sprintf(
-		"(up|lw|ti|win|mac|di|(?:dt\\.(%s)))",
+		"(up|lw|ti|win|mac|di|norm|(?:dt\\.(%s)))",
 		tokenString,
 	)
 

diff --git a/replace/variables/variables.go b/replace/variables/variables.go
@@ -804,6 +804,13 @@ func transformString(source, token string) string {
 			return source
 		}
 
+		return result
+	case "norm":
+		result, _, err := transform.String(norm.NFKC, source)
+		if err != nil {
+			return source
+		}
+
 		return result
 	}