work on pip package

This commit is contained in:
mrfakename
2025-05-06 18:59:32 -07:00
parent 54da683d36
commit a5746eaab6
36 changed files with 2928 additions and 1251 deletions
+28 -25
View File
@@ -2,39 +2,42 @@ import re
from opencc import OpenCC
t2s_converter = OpenCC('t2s')
s2t_converter = OpenCC('s2t')
t2s_converter = OpenCC("t2s")
s2t_converter = OpenCC("s2t")
EMOJI_PATTERN = re.compile(
"["
"\U0001F600-\U0001F64F" # Emoticons
"]+", flags=re.UNICODE
"\U0001f600-\U0001f64f" # Emoticons
"]+",
flags=re.UNICODE,
)
# 创建一个翻译表,用于替换和移除字符
TRANSLATION_TABLE = str.maketrans({
'-': ' ', # 将 '-' 替换为空格
',': None,
'.': None,
'': None,
'': None,
'!': None,
'': None,
'?': None,
'': None,
'': None,
';': None,
'': None,
':': None,
'': None,
'\u3000': ' ', # 将全角空格替换为空格
})
TRANSLATION_TABLE = str.maketrans(
{
"-": " ", # 将 '-' 替换为空格
",": None,
".": None,
"": None,
"": None,
"!": None,
"": None,
"?": None,
"": None,
"": None,
";": None,
"": None,
":": None,
"": None,
"\u3000": " ", # 将全角空格替换为空格
}
)
# 替换括号中的内容,包括中括号和小括号
BACKSLASH_PATTERN = re.compile(r'\(.*?\)|\[.*?\]')
BACKSLASH_PATTERN = re.compile(r"\(.*?\)|\[.*?\]")
SPACE_PATTERN = re.compile('(?<!^)\s+(?!$)')
SPACE_PATTERN = re.compile("(?<!^)\s+(?!$)")
def normalize_text(text, language, strip=True):
@@ -45,10 +48,10 @@ def normalize_text(text, language, strip=True):
text = text.translate(TRANSLATION_TABLE)
# Step 2: 移除表情符号
text = EMOJI_PATTERN.sub('', text)
text = EMOJI_PATTERN.sub("", text)
# Step 3: 连续空白字符替换为单个空格,首位除外
text = SPACE_PATTERN.sub(' ', text)
text = SPACE_PATTERN.sub(" ", text)
# Step 4: 去除首尾空白字符(如果需要)
if strip: