Skip to content

Commit

Permalink
fix: remove links from the text to prevent unnecessary processing
Browse files Browse the repository at this point in the history
  • Loading branch information
ihategfw committed Dec 19, 2024
1 parent a803c5e commit 50e04b3
Showing 1 changed file with 6 additions and 6 deletions.
12 changes: 6 additions & 6 deletions teletrans.py
Original file line number Diff line number Diff line change
Expand Up @@ -122,31 +122,31 @@ def remove_links(text):
# regrex pattern for URL
url_pattern = r'http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\\(\\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+'
# use re.sub to remove the URL from the text
return re.sub(url_pattern, '', text)
return re.sub(url_pattern, '', text).strip()


async def translate_text(text, source_lang, target_langs) -> {}:
result = {}
text = remove_links(text).strip()
if emoji.purely_emoji(text):
return result
detect_lang = detector.detect_language_of(text).iso_code_639_1.name.lower()
if detect_lang in target_langs and detect_lang != source_lang:
return result
async with aiohttp.ClientSession() as session:
tasks = []
text_without_link = remove_links(text)
for target_lang in target_langs:
if source_lang == target_lang:
result[target_lang] = text
continue
if translation_service == 'openai':
tasks.append(translate_openai(text, source_lang, target_lang, session))
tasks.append(translate_openai(text_without_link, source_lang, target_lang, session))
elif translation_service == 'google':
tasks.append(translate_google(text, source_lang, target_lang, session))
tasks.append(translate_google(text_without_link, source_lang, target_lang, session))
elif translation_service == 'azure':
tasks.append(translate_azure(text, source_lang, target_lang, session))
tasks.append(translate_azure(text_without_link, source_lang, target_lang, session))
elif translation_service == 'deeplx':
tasks.append(translate_deeplx(text, source_lang, target_lang, session))
tasks.append(translate_deeplx(text_without_link, source_lang, target_lang, session))
else:
raise Exception(
f"Unknown translation service: {translation_service}. Available services: openai, google, azure, deeplx")
Expand Down

0 comments on commit 50e04b3

Please sign in to comment.