Fix profanity filter

This commit is contained in:
2025-11-27 21:15:21 +03:00
Unverified
parent 2f12bfc412
commit add674487c
+39 -27
View File
@@ -15,7 +15,7 @@ BLOCKLIST_PATH.parent.mkdir(parents=True, exist_ok=True)
_CUSTOM_RU_TERMS: Set[str] = { _CUSTOM_RU_TERMS: Set[str] = {
"бляд", "блять", "бля", "сука", "суки", "сучка", "мразь", "ебан", "бляд", "блять", "бля", "сука", "суки", "сучка", "мразь", "ебан",
"ебать", "ебёт", "ебет", "ебаная", "ебаная", "уёбок", "уебок", "уебище", "пизда", "ебать", "ебёт", "ебет", "ебаная", "ебаная", "уёбок", "уебок", "уебище", "пизда",
"пиздец", "пизд", "хуй", "хуя", "хуе", "хуё", "хуйня", "хер", "гондон", "пиздец", "хуй", "хуя", "хуе", "хуё", "хуйня", "хер", "гондон",
"долбоёб", "долбоеб", "дебил", "член", "проститутка", "проститутки", "долбоёб", "долбоеб", "дебил", "член", "проститутка", "проститутки",
"урод", "хуесос", "хуесосы", "хуесосов", "хуесоса", "пидор", "урод", "хуесос", "хуесосы", "хуесосов", "хуесоса", "пидор",
"пидоры", "пидорас", "пидорасы", "пидорасов", "пидоры", "пидорас", "пидорасы", "пидорасов",
@@ -317,32 +317,44 @@ def _check_profanity_substrings(normalized_text: str, profane_words: Set[str]) -
# Also check if profane word appears as a subsequence (allowing extra chars) # Also check if profane word appears as a subsequence (allowing extra chars)
# This catches cases like "хуй" in "хууй" or "хU★уй" -> "хууй" # This catches cases like "хуй" in "хууй" or "хU★уй" -> "хууй"
word_chars = list(word_lower) # Only do subsequence matching for words of length 4 or more to avoid false positives
text_chars = list(normalized_lower) # Use stricter span limits for shorter words to prevent false matches in long legitimate words
if len(word_lower) >= 4:
# Try to find the word as a subsequence word_chars = list(word_lower)
i = 0 # position in text text_chars = list(normalized_lower)
j = 0 # position in word # Stricter ratio for shorter words, more lenient for longer words
seq_start = None if len(word_lower) <= 5:
max_span_ratio = 1.5 # Very strict for short words
while i < len(text_chars) and j < len(word_chars): else:
if text_chars[i] == word_chars[j]: max_span_ratio = 2.0 # Slightly more lenient for longer words
if seq_start is None:
seq_start = i # Try to find the word as a subsequence
j += 1 i = 0 # position in text
if j == len(word_chars): j = 0 # position in word
# Found the word as subsequence seq_start = None
seq_end = i + 1
# Only add if it's not already covered by exact match while i < len(text_chars) and j < len(word_chars):
if (seq_start, seq_end) not in spans: if text_chars[i] == word_chars[j]:
spans.append((seq_start, seq_end)) if seq_start is None:
# Reset to find next occurrence seq_start = i
seq_start = None j += 1
j = 0 if j == len(word_chars):
# Continue from after the start position # Found the word as subsequence
i = seq_start + 1 if seq_start is not None else i + 1 seq_end = i + 1
continue # Check if the span is reasonable (not too long)
i += 1 span_length = seq_end - seq_start
max_allowed_span = int(len(word_lower) * max_span_ratio)
if span_length <= max_allowed_span:
# Only add if it's not already covered by exact match
if (seq_start, seq_end) not in spans:
spans.append((seq_start, seq_end))
# Reset to find next occurrence - continue from after the end of this match
next_start = seq_start + 1
seq_start = None
j = 0
i = next_start
continue
i += 1
return spans return spans