mirror of
https://github.com/fromchat-messenger/web.git
synced 2026-09-22 19:15:08 +03:00
Add profanity in display and usernames, fix Docker setup
This commit is contained in:
@@ -145,10 +145,28 @@ def _render_security(action: str, fields: Dict[str, Any]) -> List[str]:
|
||||
if action == "auto_suspension_public_spam":
|
||||
lines = [
|
||||
f"Automatic suspension triggered for {_format_user(fields)}",
|
||||
f"Similar messages detected: {fields.get('similar_messages')}",
|
||||
]
|
||||
match_type = fields.get("match_type")
|
||||
if match_type:
|
||||
lines.append(f"Match type: {match_type}")
|
||||
similar = fields.get("similar_messages")
|
||||
occurrences = fields.get("occurrences")
|
||||
if similar:
|
||||
lines.append(f"Similar messages detected: {similar}")
|
||||
if occurrences and not similar:
|
||||
lines.append(f"Occurrences: {occurrences}")
|
||||
if fields.get("window_seconds"):
|
||||
lines.append(f"Observation window: {fields['window_seconds']} seconds")
|
||||
if fields.get("reason"):
|
||||
lines.append(f"Reason: {fields['reason']}")
|
||||
return lines
|
||||
if action == "auto_suspension_public_burst":
|
||||
lines = [
|
||||
f"Automatic suspension triggered for {_format_user(fields)}",
|
||||
f"Messages sent: {fields.get('count')} within {fields.get('window_seconds')} seconds",
|
||||
]
|
||||
if fields.get("reason"):
|
||||
lines.append(f"Reason: {fields['reason']}")
|
||||
return lines
|
||||
if action == "public_message_burst":
|
||||
return [
|
||||
|
||||
@@ -15,7 +15,9 @@ _CUSTOM_RU_TERMS: Set[str] = {
|
||||
"бляд", "блять", "бля", "сука", "суки", "сучка", "мразь", "ебан",
|
||||
"ебать", "ебёт", "ебет", "уёбок", "уебок", "уебище", "пизда",
|
||||
"пиздец", "пизд", "хуй", "хуя", "хуе", "хуё", "хер", "гондон",
|
||||
"долбоёб", "долбоеб", "дебил", "член", "проститутка", "урод",
|
||||
"долбоёб", "долбоеб", "дебил", "член", "проститутка", "проститутки",
|
||||
"урод", "хуесос", "хуесосы", "хуесосов", "хуесоса", "пидор",
|
||||
"пидоры", "пидорас", "пидорасы", "пидорасов",
|
||||
}
|
||||
|
||||
_ADULT_TERMS: Set[str] = {
|
||||
@@ -33,8 +35,167 @@ _PHRASE_PATTERNS: Tuple[re.Pattern[str], ...] = (
|
||||
re.compile(r"\bфромчат\s+г[ао]вно\b", re.IGNORECASE | re.UNICODE),
|
||||
re.compile(r"\b18\+\b", re.IGNORECASE | re.UNICODE),
|
||||
re.compile(r"\bxxx\b", re.IGNORECASE | re.UNICODE),
|
||||
re.compile(r"\bайфон\s+топ\b", re.IGNORECASE | re.UNICODE),
|
||||
re.compile(r"\bсамсунг\s+г[ао]вно\b", re.IGNORECASE | re.UNICODE),
|
||||
)
|
||||
|
||||
_LEET_MAP = {
|
||||
"0": "о",
|
||||
"o": "о",
|
||||
"о": "о",
|
||||
"a": "а",
|
||||
"@": "а",
|
||||
"4": "а",
|
||||
"а": "а",
|
||||
"e": "е",
|
||||
"ё": "е",
|
||||
"3": "е",
|
||||
"c": "с",
|
||||
"s": "с",
|
||||
"с": "с",
|
||||
"x": "х",
|
||||
"х": "х",
|
||||
"t": "т",
|
||||
"т": "т",
|
||||
"p": "п",
|
||||
"п": "п",
|
||||
"n": "н",
|
||||
"н": "н",
|
||||
"m": "м",
|
||||
"м": "м",
|
||||
"y": "у",
|
||||
"u": "у",
|
||||
"у": "у",
|
||||
"g": "г",
|
||||
"г": "г",
|
||||
"v": "в",
|
||||
"в": "в",
|
||||
"f": "ф",
|
||||
"ф": "ф",
|
||||
"i": "и",
|
||||
"1": "и",
|
||||
"и": "и",
|
||||
}
|
||||
|
||||
_RAW_PHRASE_GROUPS: Tuple[Tuple[str, Tuple[str, ...]], ...] = (
|
||||
("generic", ("айфон", "топ")),
|
||||
("generic", ("самсунг", "говно")),
|
||||
)
|
||||
|
||||
_SENSITIVE_PHRASE_PATH = Path("data/profanity/sensitive_phrases.json")
|
||||
_PHRASE_CACHE: dict[str, Tuple[Tuple[str, ...], ...]] = {}
|
||||
|
||||
|
||||
def _normalize_char(ch: str) -> str:
|
||||
lower = ch.lower()
|
||||
return _LEET_MAP.get(lower, lower)
|
||||
|
||||
|
||||
def _normalize_token(token: str) -> str:
|
||||
return "".join(_normalize_char(ch) for ch in token)
|
||||
|
||||
|
||||
def _tokenize_with_spans(text: str) -> List[Tuple[int, int, str]]:
|
||||
tokens: List[Tuple[int, int, str]] = []
|
||||
start: int | None = None
|
||||
buffer: List[str] = []
|
||||
|
||||
for idx, ch in enumerate(text):
|
||||
if ch.isalnum() or ch in {"@", "#", "_"}:
|
||||
if start is None:
|
||||
start = idx
|
||||
buffer.append(ch)
|
||||
else:
|
||||
if buffer and start is not None:
|
||||
token_raw = "".join(buffer)
|
||||
tokens.append((start, idx, _normalize_token(token_raw)))
|
||||
buffer.clear()
|
||||
start = None
|
||||
if buffer and start is not None:
|
||||
token_raw = "".join(buffer)
|
||||
tokens.append((start, len(text), _normalize_token(token_raw)))
|
||||
return tokens
|
||||
|
||||
|
||||
def _edit_distance_limited(a: str, b: str, max_distance: int = 1) -> bool:
|
||||
if a == b:
|
||||
return True
|
||||
if max_distance <= 0:
|
||||
return False
|
||||
if abs(len(a) - len(b)) > max_distance:
|
||||
return False
|
||||
|
||||
previous = list(range(len(b) + 1))
|
||||
for i, ca in enumerate(a, 1):
|
||||
current = [i]
|
||||
best = current[0]
|
||||
for j, cb in enumerate(b, 1):
|
||||
insert_cost = current[j - 1] + 1
|
||||
delete_cost = previous[j] + 1
|
||||
replace_cost = previous[j - 1] + (0 if ca == cb else 1)
|
||||
cost = min(insert_cost, delete_cost, replace_cost)
|
||||
current.append(cost)
|
||||
if cost < best:
|
||||
best = cost
|
||||
if best > max_distance:
|
||||
return False
|
||||
previous = current
|
||||
return previous[-1] <= max_distance
|
||||
|
||||
|
||||
def _load_sensitive_phrases() -> List[Tuple[str, ...]]:
|
||||
if not _SENSITIVE_PHRASE_PATH.exists():
|
||||
return []
|
||||
try:
|
||||
payload = json.loads(_SENSITIVE_PHRASE_PATH.read_text(encoding="utf-8"))
|
||||
phrases: List[Tuple[str, ...]] = []
|
||||
if isinstance(payload, list):
|
||||
for entry in payload:
|
||||
if isinstance(entry, list) and entry:
|
||||
normalized = tuple(str(part).strip() for part in entry if str(part).strip())
|
||||
if normalized:
|
||||
phrases.append(normalized)
|
||||
return phrases
|
||||
except Exception:
|
||||
return []
|
||||
|
||||
|
||||
def _get_phrases(group: str) -> Tuple[Tuple[str, ...], ...]:
|
||||
if group not in _PHRASE_CACHE:
|
||||
base = [phrase for key, phrase in _RAW_PHRASE_GROUPS if key == group]
|
||||
if group == "sensitive":
|
||||
base.extend(_load_sensitive_phrases())
|
||||
_PHRASE_CACHE[group] = tuple(
|
||||
tuple(_normalize_token(part) for part in phrase)
|
||||
for phrase in base
|
||||
)
|
||||
return _PHRASE_CACHE[group]
|
||||
|
||||
|
||||
def _find_fuzzy_phrase_spans(text: str, group: str = "generic") -> List[Tuple[int, int]]:
|
||||
tokens = _tokenize_with_spans(text)
|
||||
if not tokens:
|
||||
return []
|
||||
|
||||
spans: List[Tuple[int, int]] = []
|
||||
normalized_phrases = _get_phrases(group)
|
||||
|
||||
for index in range(len(tokens)):
|
||||
for phrase in normalized_phrases:
|
||||
if index + len(phrase) > len(tokens):
|
||||
continue
|
||||
matches = True
|
||||
for offset, target in enumerate(phrase):
|
||||
token = tokens[index + offset][2]
|
||||
if not _edit_distance_limited(token, target):
|
||||
matches = False
|
||||
break
|
||||
if matches:
|
||||
span_start = tokens[index][0]
|
||||
span_end = tokens[index + len(phrase) - 1][1]
|
||||
spans.append((span_start, span_end))
|
||||
return spans
|
||||
|
||||
_dictionary_lock = RLock()
|
||||
_blocklist_signature: Tuple[str, ...] | None = None
|
||||
_profanity = Profanity()
|
||||
@@ -96,7 +257,11 @@ def _apply_phrase_filters(text: str) -> str:
|
||||
match = pattern.search(result)
|
||||
if not match:
|
||||
break
|
||||
result = result[:match.start()] + ("\\*" * (match.end() - match.start())) + result[match.end():]
|
||||
result = result[:match.start()] + ("*" * (match.end() - match.start())) + result[match.end():]
|
||||
|
||||
for start, end in sorted(_find_fuzzy_phrase_spans(text, "generic"), reverse=True):
|
||||
result = result[:start] + ("*" * (end - start)) + result[end:]
|
||||
|
||||
return result
|
||||
|
||||
|
||||
@@ -109,6 +274,27 @@ def censor_text(text: str) -> str:
|
||||
return _profanity.censor(preprocessed, censor_char="\\*")
|
||||
|
||||
|
||||
def contains_profanity(text: str) -> bool:
|
||||
if not text:
|
||||
return False
|
||||
|
||||
_rebuild_dictionary()
|
||||
for pattern in _PHRASE_PATTERNS:
|
||||
if pattern.search(text):
|
||||
return True
|
||||
if _find_fuzzy_phrase_spans(text, "generic"):
|
||||
return True
|
||||
return _profanity.contains_profanity(text)
|
||||
|
||||
|
||||
def contains_sensitive_phrase(text: str) -> bool:
|
||||
if not text:
|
||||
return False
|
||||
if _find_fuzzy_phrase_spans(text, "sensitive"):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def get_blocklist() -> List[str]:
|
||||
with _dictionary_lock:
|
||||
return sorted(_load_blocklist())
|
||||
|
||||
Reference in New Issue
Block a user