mirror of
https://github.com/fromchat-messenger/web.git
synced 2026-09-22 19:15:08 +03:00
Fix profanity filter
This commit is contained in:
@@ -15,7 +15,7 @@ BLOCKLIST_PATH.parent.mkdir(parents=True, exist_ok=True)
|
|||||||
_CUSTOM_RU_TERMS: Set[str] = {
|
_CUSTOM_RU_TERMS: Set[str] = {
|
||||||
"бляд", "блять", "бля", "сука", "суки", "сучка", "мразь", "ебан",
|
"бляд", "блять", "бля", "сука", "суки", "сучка", "мразь", "ебан",
|
||||||
"ебать", "ебёт", "ебет", "ебаная", "ебаная", "уёбок", "уебок", "уебище", "пизда",
|
"ебать", "ебёт", "ебет", "ебаная", "ебаная", "уёбок", "уебок", "уебище", "пизда",
|
||||||
"пиздец", "пизд", "хуй", "хуя", "хуе", "хуё", "хуйня", "хер", "гондон",
|
"пиздец", "хуй", "хуя", "хуе", "хуё", "хуйня", "хер", "гондон",
|
||||||
"долбоёб", "долбоеб", "дебил", "член", "проститутка", "проститутки",
|
"долбоёб", "долбоеб", "дебил", "член", "проститутка", "проститутки",
|
||||||
"урод", "хуесос", "хуесосы", "хуесосов", "хуесоса", "пидор",
|
"урод", "хуесос", "хуесосы", "хуесосов", "хуесоса", "пидор",
|
||||||
"пидоры", "пидорас", "пидорасы", "пидорасов",
|
"пидоры", "пидорас", "пидорасы", "пидорасов",
|
||||||
@@ -317,8 +317,16 @@ def _check_profanity_substrings(normalized_text: str, profane_words: Set[str]) -
|
|||||||
|
|
||||||
# Also check if profane word appears as a subsequence (allowing extra chars)
|
# Also check if profane word appears as a subsequence (allowing extra chars)
|
||||||
# This catches cases like "хуй" in "хууй" or "хU★уй" -> "хууй"
|
# This catches cases like "хуй" in "хууй" or "хU★уй" -> "хууй"
|
||||||
|
# Only do subsequence matching for words of length 4 or more to avoid false positives
|
||||||
|
# Use stricter span limits for shorter words to prevent false matches in long legitimate words
|
||||||
|
if len(word_lower) >= 4:
|
||||||
word_chars = list(word_lower)
|
word_chars = list(word_lower)
|
||||||
text_chars = list(normalized_lower)
|
text_chars = list(normalized_lower)
|
||||||
|
# Stricter ratio for shorter words, more lenient for longer words
|
||||||
|
if len(word_lower) <= 5:
|
||||||
|
max_span_ratio = 1.5 # Very strict for short words
|
||||||
|
else:
|
||||||
|
max_span_ratio = 2.0 # Slightly more lenient for longer words
|
||||||
|
|
||||||
# Try to find the word as a subsequence
|
# Try to find the word as a subsequence
|
||||||
i = 0 # position in text
|
i = 0 # position in text
|
||||||
@@ -333,14 +341,18 @@ def _check_profanity_substrings(normalized_text: str, profane_words: Set[str]) -
|
|||||||
if j == len(word_chars):
|
if j == len(word_chars):
|
||||||
# Found the word as subsequence
|
# Found the word as subsequence
|
||||||
seq_end = i + 1
|
seq_end = i + 1
|
||||||
|
# Check if the span is reasonable (not too long)
|
||||||
|
span_length = seq_end - seq_start
|
||||||
|
max_allowed_span = int(len(word_lower) * max_span_ratio)
|
||||||
|
if span_length <= max_allowed_span:
|
||||||
# Only add if it's not already covered by exact match
|
# Only add if it's not already covered by exact match
|
||||||
if (seq_start, seq_end) not in spans:
|
if (seq_start, seq_end) not in spans:
|
||||||
spans.append((seq_start, seq_end))
|
spans.append((seq_start, seq_end))
|
||||||
# Reset to find next occurrence
|
# Reset to find next occurrence - continue from after the end of this match
|
||||||
|
next_start = seq_start + 1
|
||||||
seq_start = None
|
seq_start = None
|
||||||
j = 0
|
j = 0
|
||||||
# Continue from after the start position
|
i = next_start
|
||||||
i = seq_start + 1 if seq_start is not None else i + 1
|
|
||||||
continue
|
continue
|
||||||
i += 1
|
i += 1
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user