mirror of
https://github.com/fromchat-messenger/web.git
synced 2026-09-22 19:15:08 +03:00
Restructure backend into microservices, add envelope encryption, DM files, and message editing
This commit is contained in:
@@ -0,0 +1,2 @@
|
||||
# Package marker for security utilities
|
||||
|
||||
@@ -0,0 +1,406 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from html import unescape
|
||||
from typing import Any, Callable, Dict, List
|
||||
|
||||
from ..logging_config import access_logger, dm_logger, public_chat_logger, security_logger
|
||||
|
||||
|
||||
def _clean_username(username: Any) -> str:
|
||||
if not username:
|
||||
return "unknown user"
|
||||
return f"@{username}"
|
||||
|
||||
|
||||
def _format_user(fields: Dict[str, Any], username_key: str = "username", user_id_key: str = "user_id") -> str:
|
||||
username = fields.get(username_key)
|
||||
if username is None and "_" in username_key:
|
||||
base_key = username_key.split("_", 1)[0]
|
||||
username = fields.get(base_key)
|
||||
|
||||
user_id = fields.get(user_id_key)
|
||||
if user_id is None and "_" in user_id_key:
|
||||
base_key = user_id_key.split("_", 1)[0]
|
||||
user_id = fields.get(base_key)
|
||||
|
||||
if username and user_id is not None:
|
||||
return f"{_clean_username(username)} (user id {user_id})"
|
||||
if username:
|
||||
return _clean_username(username)
|
||||
if user_id is not None:
|
||||
return f"user id {user_id}"
|
||||
return "unknown user"
|
||||
|
||||
|
||||
def _format_actor(fields: Dict[str, Any], prefix: str) -> str:
|
||||
return _format_user(fields, f"{prefix}_username", f"{prefix}_id")
|
||||
|
||||
|
||||
def _plural(label: str, count: int) -> str:
|
||||
return f"{count} {label if count == 1 else label + 's'}"
|
||||
|
||||
|
||||
def _yes_no(flag: Any) -> str:
|
||||
return "yes" if flag else "no"
|
||||
|
||||
|
||||
def _render_security(action: str, fields: Dict[str, Any]) -> List[str]:
|
||||
if action == "login_success":
|
||||
lines = [f"Login approved for {_format_user(fields)}"]
|
||||
session = fields.get("session_id")
|
||||
if session:
|
||||
lines.append(f"Session: {session}")
|
||||
client_bits: List[str] = []
|
||||
if fields.get("device"):
|
||||
client_bits.append(fields["device"])
|
||||
if fields.get("os"):
|
||||
client_bits.append(fields["os"])
|
||||
if fields.get("browser"):
|
||||
client_bits.append(fields["browser"])
|
||||
if client_bits:
|
||||
lines.append(f"Client: {', '.join(client_bits)}")
|
||||
if fields.get("ip"):
|
||||
lines.append(f"IP address: {fields['ip']}")
|
||||
return lines
|
||||
if action == "login_failed":
|
||||
lines = [f"Login denied for {_format_user(fields)}"]
|
||||
if fields.get("reason"):
|
||||
lines.append(f"Reason: {fields['reason']}")
|
||||
if fields.get("ip"):
|
||||
lines.append(f"IP address: {fields['ip']}")
|
||||
return lines
|
||||
if action == "auth_bruteforce_detected":
|
||||
lines = ["Brute-force login pattern detected"]
|
||||
lines.append(f"Target: {_format_user(fields)}")
|
||||
failures = fields.get("failures")
|
||||
if isinstance(failures, dict):
|
||||
for key, value in failures.items():
|
||||
lines.append(f"{key}: {value}")
|
||||
if fields.get("ip"):
|
||||
lines.append(f"IP address: {fields['ip']}")
|
||||
if fields.get("window_seconds"):
|
||||
lines.append(f"Observation window: {fields['window_seconds']} seconds")
|
||||
return lines
|
||||
if action == "registration_success":
|
||||
ip_raw = fields.get("ip")
|
||||
ip_display = "localhost" if ip_raw in {"127.0.0.1", "::1"} else ip_raw
|
||||
display_name = fields.get("display_name") or "Unknown"
|
||||
username = fields.get("username")
|
||||
user_id = fields.get("user_id")
|
||||
user_agent = fields.get("user_agent") or "Unknown user agent"
|
||||
lines = ["Account registered"]
|
||||
lines.append(f"Display name: {display_name}")
|
||||
lines.append(f"Username: {_clean_username(username) if username else 'unknown'}")
|
||||
if ip_display:
|
||||
lines.append(f"IP: {ip_display}")
|
||||
if user_agent:
|
||||
lines.append(f"User agent: {user_agent}")
|
||||
if user_id is not None:
|
||||
lines.append(f"User ID: {user_id}")
|
||||
return lines
|
||||
if action == "password_changed":
|
||||
lines = [f"Password changed for {_format_user(fields)}"]
|
||||
lines.append(f"Other sessions revoked: {_yes_no(fields.get('logout_others'))}")
|
||||
if fields.get("ip"):
|
||||
lines.append(f"IP address: {fields['ip']}")
|
||||
return lines
|
||||
if action == "logout":
|
||||
lines = [f"Logout recorded for {_format_user(fields)}"]
|
||||
if fields.get("session_id"):
|
||||
lines.append(f"Session: {fields['session_id']}")
|
||||
if fields.get("ip"):
|
||||
lines.append(f"IP address: {fields['ip']}")
|
||||
return lines
|
||||
if action == "admin_delete_user":
|
||||
return [
|
||||
"Account removal",
|
||||
f"Actor: {_format_actor(fields, 'actor')}",
|
||||
f"Target: {_format_actor(fields, 'target')}",
|
||||
]
|
||||
if action == "admin_suspend_user":
|
||||
lines = [
|
||||
"User suspension",
|
||||
f"Actor: {_format_actor(fields, 'actor')}",
|
||||
f"Target: {_format_actor(fields, 'target')}",
|
||||
]
|
||||
if fields.get("reason"):
|
||||
lines.append(f"Reason: {fields.get('reason')}")
|
||||
return lines
|
||||
if action == "admin_unsuspend_user":
|
||||
return [
|
||||
"User unsuspension",
|
||||
f"Actor: {_format_actor(fields, 'actor')}",
|
||||
f"Target: {_format_actor(fields, 'target')}",
|
||||
]
|
||||
if action == "admin_verify_toggle":
|
||||
return [
|
||||
"User verification",
|
||||
f"Actor: {_format_actor(fields, 'actor')}",
|
||||
f"Target: {_format_actor(fields, 'target')}",
|
||||
f"Verified: {_yes_no(fields.get('verified'))}",
|
||||
]
|
||||
if action == "self_delete_account":
|
||||
return [f"User {_format_user(fields)} deleted their account"]
|
||||
if action == "auto_suspension_public_spam":
|
||||
lines = [
|
||||
f"Automatic suspension triggered for {_format_user(fields)}",
|
||||
]
|
||||
match_type = fields.get("match_type")
|
||||
if match_type:
|
||||
lines.append(f"Match type: {match_type}")
|
||||
similar = fields.get("similar_messages")
|
||||
occurrences = fields.get("occurrences")
|
||||
if similar:
|
||||
lines.append(f"Similar messages detected: {similar}")
|
||||
if occurrences and not similar:
|
||||
lines.append(f"Occurrences: {occurrences}")
|
||||
if fields.get("window_seconds"):
|
||||
lines.append(f"Observation window: {fields['window_seconds']} seconds")
|
||||
if fields.get("reason"):
|
||||
lines.append(f"Reason: {fields['reason']}")
|
||||
return lines
|
||||
if action == "auto_suspension_public_burst":
|
||||
lines = [
|
||||
f"Automatic suspension triggered for {_format_user(fields)}",
|
||||
f"Messages sent: {fields.get('count')} within {fields.get('window_seconds')} seconds",
|
||||
]
|
||||
if fields.get("reason"):
|
||||
lines.append(f"Reason: {fields['reason']}")
|
||||
return lines
|
||||
if action == "public_message_burst":
|
||||
return [
|
||||
f"Rapid messaging spike for {_format_user(fields)}",
|
||||
f"Messages sent: {fields.get('count')} within {fields.get('window_seconds')} seconds",
|
||||
]
|
||||
if action == "blocklist_add":
|
||||
added = fields.get("added") or []
|
||||
lines = [f"Blocklist updated by {_format_actor(fields, 'actor')}"]
|
||||
if added:
|
||||
lines.append(f"Added entries: {', '.join(added)}")
|
||||
total = len(fields.get("words") or [])
|
||||
lines.append(f"Total entries: {total}")
|
||||
return lines
|
||||
if action == "blocklist_remove":
|
||||
removed = fields.get("removed") or []
|
||||
lines = [f"Blocklist cleaned by {_format_actor(fields, 'actor')}"]
|
||||
if removed:
|
||||
lines.append(f"Removed entries: {', '.join(removed)}")
|
||||
total = len(fields.get("words") or [])
|
||||
lines.append(f"Total entries: {total}")
|
||||
return lines
|
||||
return [f"{action.replace('_', ' ').capitalize()}"] + [
|
||||
f"{key.replace('_', ' ').capitalize()}: {value}"
|
||||
for key, value in fields.items()
|
||||
if value is not None
|
||||
]
|
||||
|
||||
|
||||
def _render_public_chat(action: str, fields: Dict[str, Any]) -> List[str]:
|
||||
if action == "message_created":
|
||||
lines = [f"Message #{fields.get('message_id')} sent by {_format_user(fields)}"]
|
||||
if fields.get("reply_to"):
|
||||
lines.append(f"In reply to message #{fields['reply_to']}")
|
||||
attachments = fields.get("attachments")
|
||||
if attachments:
|
||||
lines.append(f"Attachments: {_plural('file', attachments)}")
|
||||
|
||||
# If content was censored, log both raw and censored versions
|
||||
if fields.get("raw_content") is not None:
|
||||
lines.append("Raw content (before censoring):")
|
||||
for line in unescape(fields["raw_content"]).splitlines():
|
||||
lines.append(f"| {line}")
|
||||
lines.append("Censored content (stored):")
|
||||
for line in unescape(fields.get("censored_content", fields.get("content", ""))).splitlines():
|
||||
lines.append(f"| {line}")
|
||||
elif fields.get("content"):
|
||||
lines.append("Content:")
|
||||
for line in unescape(fields["content"]).splitlines():
|
||||
lines.append(f"| {line}")
|
||||
return lines
|
||||
if action == "message_edited":
|
||||
lines = [f"Message #{fields.get('message_id')} edited by {_format_user(fields)}"]
|
||||
if fields.get("reply_to"):
|
||||
lines.append(f"Reply to #{fields['reply_to']}")
|
||||
if fields.get("previous_content"):
|
||||
lines.append("Previous content:")
|
||||
for line in unescape(fields["previous_content"] or "").splitlines() or [""]:
|
||||
lines.append(f"| {line}")
|
||||
|
||||
# If content was censored, log both raw and censored versions
|
||||
if fields.get("raw_content") is not None:
|
||||
lines.append("Raw content (before censoring):")
|
||||
for line in unescape(fields["raw_content"]).splitlines():
|
||||
lines.append(f"| {line}")
|
||||
lines.append("Censored content (stored):")
|
||||
for line in unescape(fields.get("censored_content", fields.get("content", ""))).splitlines():
|
||||
lines.append(f"| {line}")
|
||||
elif fields.get("content"):
|
||||
lines.append("New content:")
|
||||
for line in unescape(fields["content"] or "").splitlines() or [""]:
|
||||
lines.append(f"| {line}")
|
||||
|
||||
return lines
|
||||
if action == "message_deleted":
|
||||
lines = [
|
||||
f"Message #{fields.get('message_id')} deleted",
|
||||
f"Actor: {_format_actor(fields, 'actor')}",
|
||||
]
|
||||
if fields.get("original_author_id") is not None:
|
||||
lines.append(f"Original author: user #{fields['original_author_id']}")
|
||||
if fields.get("content"):
|
||||
lines.append("Previous content:")
|
||||
for line in unescape(fields["content"]).splitlines():
|
||||
lines.append(f"| {line}")
|
||||
return lines
|
||||
if action == "reaction_update":
|
||||
lines = [
|
||||
f"Reaction {fields.get('action', 'updated')} on message #{fields.get('message_id')}",
|
||||
f"User: {_format_user(fields)}",
|
||||
]
|
||||
if fields.get("emoji"):
|
||||
lines.append(f"Emoji: {fields['emoji']}")
|
||||
return lines
|
||||
return [f"{action.replace('_', ' ').capitalize()}"] + [
|
||||
f"{key.replace('_', ' ').capitalize()}: {value}"
|
||||
for key, value in fields.items()
|
||||
if value is not None
|
||||
]
|
||||
|
||||
|
||||
def _render_dm(action: str, fields: Dict[str, Any]) -> List[str]:
|
||||
if action in {"message_sent", "message_sent_ws"}:
|
||||
lines = [
|
||||
f"Direct message #{fields.get('dm_envelope_id')} sent",
|
||||
f"Sender: {_format_actor(fields, 'sender')}",
|
||||
]
|
||||
if fields.get("recipient_id") is not None:
|
||||
lines.append(f"Recipient: user id {fields['recipient_id']}")
|
||||
attachments = fields.get("attachment_count")
|
||||
if attachments:
|
||||
lines.append(f"Attachments: {_plural('file', attachments)}")
|
||||
if fields.get("reply_to"):
|
||||
lines.append(f"In reply to DM #{fields['reply_to']}")
|
||||
return lines
|
||||
if action == "message_edited":
|
||||
return [
|
||||
f"Direct message #{fields.get('dm_envelope_id')} edited",
|
||||
f"Author: {_format_user(fields)}",
|
||||
]
|
||||
if action == "message_deleted":
|
||||
lines = [
|
||||
f"Direct message #{fields.get('dm_envelope_id')} deleted",
|
||||
f"Actor: {_format_user(fields)}",
|
||||
]
|
||||
if fields.get("recipient_id") is not None:
|
||||
lines.append(f"Recipient: user id {fields['recipient_id']}")
|
||||
return lines
|
||||
if action == "reaction_update":
|
||||
lines = [
|
||||
f"Reaction {fields.get('action', 'updated')} on DM #{fields.get('dm_envelope_id')}",
|
||||
f"User: {_format_user(fields)}",
|
||||
]
|
||||
if fields.get("emoji"):
|
||||
lines.append(f"Emoji: {fields['emoji']}")
|
||||
return lines
|
||||
return [f"{action.replace('_', ' ').capitalize()}"] + [
|
||||
f"{key.replace('_', ' ').capitalize()}: {value}"
|
||||
for key, value in fields.items()
|
||||
if value is not None
|
||||
]
|
||||
|
||||
|
||||
def _render_access(action: str, fields: Dict[str, Any]) -> List[str]:
|
||||
ip_raw = fields.get("ip")
|
||||
ip_display = "localhost" if ip_raw in {"127.0.0.1", "::1"} else ip_raw
|
||||
if action == "http_request":
|
||||
first_line = f"{fields.get('method')} {fields.get('path')}"
|
||||
if ip_display:
|
||||
first_line += f" from {ip_display}"
|
||||
first_line += f" -> {fields.get('status')}"
|
||||
lines = [first_line]
|
||||
if fields.get("user"):
|
||||
lines.append(f"Authenticated user: {_clean_username(fields['user'])}")
|
||||
return lines
|
||||
if action == "http_error":
|
||||
first_line = f"HTTP error during {fields.get('method')} {fields.get('path')}"
|
||||
if ip_display:
|
||||
first_line += f" from {ip_display}"
|
||||
lines = [first_line]
|
||||
if fields.get("error"):
|
||||
lines.append(f"Exception: {fields['error']}")
|
||||
if fields.get("user"):
|
||||
lines.append(f"Authenticated user: {_clean_username(fields['user'])}")
|
||||
return lines
|
||||
if action == "ws_connect":
|
||||
lines = ["WebSocket connected"]
|
||||
if fields.get("path"):
|
||||
lines.append(f"Endpoint: {fields['path']}")
|
||||
if ip_display:
|
||||
lines.append(f"IP: {ip_display}")
|
||||
return lines
|
||||
if action == "ws_disconnect":
|
||||
lines = ["WebSocket disconnected"]
|
||||
if fields.get("path"):
|
||||
lines.append(f"Endpoint: {fields['path']}")
|
||||
if fields.get("code") is not None:
|
||||
reason = fields.get("reason") or "no reason"
|
||||
lines.append(f"Code {fields['code']} ({reason})")
|
||||
if ip_display:
|
||||
lines.append(f"IP: {ip_display}")
|
||||
return lines
|
||||
if action == "ws_event":
|
||||
event_name = fields.get("event")
|
||||
path = fields.get("path")
|
||||
first_line = "WS"
|
||||
if path:
|
||||
first_line += f" {path}"
|
||||
if ip_display:
|
||||
first_line += f" from {ip_display}"
|
||||
if event_name:
|
||||
first_line += f" -> {event_name}"
|
||||
lines = [first_line]
|
||||
if fields.get("user"):
|
||||
lines.append(f"Authenticated user: {_format_user(fields, 'user', 'user_id')}")
|
||||
for key, value in fields.items():
|
||||
if key in {"path", "event", "user", "user_id", "ip"} or value is None:
|
||||
continue
|
||||
lines.append(f"{key.replace('_', ' ').capitalize()}: {value}")
|
||||
return lines
|
||||
return [f"{action.replace('_', ' ').capitalize()}"] + [
|
||||
f"{key.replace('_', ' ').capitalize()}: {value}"
|
||||
for key, value in fields.items()
|
||||
if value is not None
|
||||
]
|
||||
|
||||
|
||||
def _log_event(
|
||||
logger: logging.Logger,
|
||||
renderer: Callable[[str, Dict[str, Any]], List[str]],
|
||||
action: str,
|
||||
severity: str,
|
||||
fields: Dict[str, Any],
|
||||
) -> None:
|
||||
lines = renderer(action, fields)
|
||||
if not lines:
|
||||
return
|
||||
level = getattr(logging, severity.upper(), logging.INFO)
|
||||
logger.log(level, "\n".join(lines))
|
||||
|
||||
|
||||
def log_security(action: str, severity: str = "info", **fields: Any) -> None:
|
||||
_log_event(security_logger, _render_security, action, severity, fields)
|
||||
|
||||
|
||||
def log_public_chat(action: str, severity: str = "info", **fields: Any) -> None:
|
||||
_log_event(public_chat_logger, _render_public_chat, action, severity, fields)
|
||||
|
||||
|
||||
def log_dm(action: str, severity: str = "info", **fields: Any) -> None:
|
||||
sanitized_fields = {key: value for key, value in fields.items() if key != "content"}
|
||||
_log_event(dm_logger, _render_dm, action, severity, sanitized_fields)
|
||||
|
||||
|
||||
def log_access(action: str, severity: str = "info", **fields: Any) -> None:
|
||||
_log_event(access_logger, _render_access, action, severity, fields)
|
||||
|
||||
@@ -0,0 +1,694 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
import unicodedata
|
||||
from pathlib import Path
|
||||
from threading import RLock
|
||||
from typing import Iterable, List, Set, Tuple
|
||||
|
||||
from better_profanity import Profanity
|
||||
|
||||
BLOCKLIST_PATH = Path("data/profanity/blocklist.json")
|
||||
BLOCKLIST_PATH.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
_CUSTOM_RU_TERMS: Set[str] = {
|
||||
"бляд", "блять", "бля", "сука", "суки", "сучка", "мразь", "ебан",
|
||||
"ебать", "ебёт", "ебет", "ебаная", "ебаная", "уёбок", "уебок", "уебище", "пизда",
|
||||
"пиздец", "хуй", "хуя", "хуе", "хуё", "хуйня", "хер", "гондон",
|
||||
"долбоёб", "долбоеб", "дебил", "член", "проститутка", "проститутки",
|
||||
"урод", "хуесос", "хуесосы", "хуесосов", "хуесоса", "сос", "пидор",
|
||||
"пидоры", "пидорас", "пидорасы", "пидорасов",
|
||||
}
|
||||
|
||||
_ADULT_TERMS: Set[str] = {
|
||||
"порно", "порнуха", "эротика", "эротический", "секс", "сексуальный",
|
||||
"инцест", "порнография", "порностудия", "порновидео", "порносайт",
|
||||
"сексчат", "сексчатик", "секслайв", "сексвидео",
|
||||
}
|
||||
|
||||
_STATIC_TERMS: Set[str] = set(term.lower() for term in (_CUSTOM_RU_TERMS | _ADULT_TERMS))
|
||||
|
||||
# Words that should never be flagged as profanity (whitelist)
|
||||
_WHITELIST: Set[str] = {
|
||||
"говно", # Allow this word
|
||||
}
|
||||
|
||||
# Phrase patterns - these will be applied to normalized text (without special chars)
|
||||
_PHRASE_PATTERNS: Tuple[re.Pattern[str], ...] = (
|
||||
re.compile(r"\bmax\s+is\s+better\b", re.IGNORECASE | re.UNICODE),
|
||||
re.compile(r"\bмакс\s+лучше\b", re.IGNORECASE | re.UNICODE),
|
||||
re.compile(r"\bfromchat\s+г[ао]вно\b", re.IGNORECASE | re.UNICODE),
|
||||
re.compile(r"\bфромчат\s+г[ао]вно\b", re.IGNORECASE | re.UNICODE),
|
||||
re.compile(r"\b18\+\b", re.IGNORECASE | re.UNICODE),
|
||||
re.compile(r"\bxxx\b", re.IGNORECASE | re.UNICODE),
|
||||
re.compile(r"\bайфон\s+топ\b", re.IGNORECASE | re.UNICODE),
|
||||
re.compile(r"\bсамсунг\s+г[ао]вно\b", re.IGNORECASE | re.UNICODE),
|
||||
)
|
||||
|
||||
# Patterns to check in original text (before normalization) to catch visual bypasses
|
||||
# These patterns check for special character combinations that visually form letters
|
||||
_ORIGINAL_TEXT_PATTERNS: Tuple[re.Pattern[str], ...] = (
|
||||
# Catch "}{" used to visually form "х" followed by "С0С" or similar patterns
|
||||
# This catches "хуесос" written as "}{¥€С0С" or variations
|
||||
# Matches: }{ + any characters (including special chars) + С/с + 0 + С/с
|
||||
# The pattern allows any characters between to catch special chars like ¥€
|
||||
re.compile(r"}\{.*?[сcСC].*?[0оoОO].*?[сcСC]", re.IGNORECASE | re.UNICODE),
|
||||
# Also catch "}{" followed by "уесос" with 0 instead of о
|
||||
re.compile(r"}\{.*?[уyУY].*?[еeЕE].*?[сcСC].*?[0оoОO].*?[сcСC]", re.IGNORECASE | re.UNICODE),
|
||||
)
|
||||
|
||||
# Map for normalizing homoglyphs (similar-looking characters)
|
||||
# Maps English/Latin characters to their Cyrillic equivalents and vice versa
|
||||
# Also includes Greek, full-width, and other Unicode variants
|
||||
_LEET_MAP = {
|
||||
# Numbers to letters
|
||||
"0": "о",
|
||||
"1": "и",
|
||||
"3": "е",
|
||||
"4": "а",
|
||||
# Latin to Cyrillic (lowercase)
|
||||
"a": "а",
|
||||
"c": "с",
|
||||
"e": "е",
|
||||
"f": "ф",
|
||||
"g": "г",
|
||||
"i": "и",
|
||||
"m": "м",
|
||||
"n": "н",
|
||||
"o": "о",
|
||||
"p": "п",
|
||||
"s": "с",
|
||||
"t": "т",
|
||||
"u": "у",
|
||||
"v": "в",
|
||||
"x": "х",
|
||||
"y": "у",
|
||||
"z": "з", # English 'z' to Cyrillic 'з'
|
||||
# Latin to Cyrillic (uppercase)
|
||||
"A": "а",
|
||||
"C": "с",
|
||||
"E": "е",
|
||||
"F": "ф",
|
||||
"G": "г",
|
||||
"I": "и",
|
||||
"M": "м",
|
||||
"N": "н",
|
||||
"O": "о",
|
||||
"P": "п",
|
||||
"S": "с",
|
||||
"T": "т",
|
||||
"U": "у",
|
||||
"V": "в",
|
||||
"X": "х",
|
||||
"Y": "у",
|
||||
"Z": "з", # English 'Z' to Cyrillic 'з'
|
||||
# Greek letters that look like Cyrillic/Latin
|
||||
"α": "а", # Greek alpha
|
||||
"Α": "а",
|
||||
"ο": "о", # Greek omicron
|
||||
"Ο": "о",
|
||||
"ρ": "р", # Greek rho (looks like Cyrillic р)
|
||||
"Ρ": "р",
|
||||
"υ": "у", # Greek upsilon
|
||||
"Υ": "у",
|
||||
"χ": "х", # Greek chi
|
||||
"Χ": "х",
|
||||
"ε": "е", # Greek epsilon
|
||||
"Ε": "е",
|
||||
"ι": "и", # Greek iota
|
||||
"Ι": "и",
|
||||
"ν": "н", # Greek nu
|
||||
"Ν": "н",
|
||||
"μ": "м", # Greek mu
|
||||
"Μ": "м",
|
||||
"π": "п", # Greek pi
|
||||
"Π": "п",
|
||||
"τ": "т", # Greek tau
|
||||
"Τ": "т",
|
||||
"γ": "г", # Greek gamma
|
||||
"Γ": "г",
|
||||
"σ": "с", # Greek sigma
|
||||
"Σ": "с",
|
||||
"φ": "ф", # Greek phi
|
||||
"Φ": "ф",
|
||||
# Full-width Latin characters
|
||||
"ａ": "а",
|
||||
"Ａ": "а",
|
||||
"ｃ": "с",
|
||||
"Ｃ": "с",
|
||||
"ｅ": "е",
|
||||
"Ｅ": "е",
|
||||
"ｆ": "ф",
|
||||
"Ｆ": "ф",
|
||||
"ｇ": "г",
|
||||
"Ｇ": "г",
|
||||
"ｉ": "и",
|
||||
"Ｉ": "и",
|
||||
"ｍ": "м",
|
||||
"Ｍ": "м",
|
||||
"ｎ": "н",
|
||||
"Ｎ": "н",
|
||||
"ｏ": "о",
|
||||
"Ｏ": "о",
|
||||
"ｐ": "п",
|
||||
"Ｐ": "п",
|
||||
"ｓ": "с",
|
||||
"Ｓ": "с",
|
||||
"ｔ": "т",
|
||||
"Ｔ": "т",
|
||||
"ｕ": "у",
|
||||
"Ｕ": "у",
|
||||
"ｖ": "в",
|
||||
"Ｖ": "в",
|
||||
"ｘ": "х",
|
||||
"Ｘ": "х",
|
||||
"ｙ": "у",
|
||||
"Ｙ": "у",
|
||||
"ｚ": "з", # Full-width 'z' to Cyrillic 'з'
|
||||
"Ｚ": "з",
|
||||
# Cyrillic to canonical Cyrillic (identity mappings)
|
||||
"а": "а",
|
||||
"с": "с",
|
||||
"е": "е",
|
||||
"ё": "е",
|
||||
"ф": "ф",
|
||||
"г": "г",
|
||||
"и": "и",
|
||||
"м": "м",
|
||||
"н": "н",
|
||||
"о": "о",
|
||||
"п": "п",
|
||||
"т": "т",
|
||||
"у": "у",
|
||||
"ү": "у", # Cyrillic capital U (U+04AE)
|
||||
"Ү": "у", # Cyrillic capital U (U+04AE)
|
||||
"в": "в",
|
||||
"х": "х",
|
||||
"р": "р",
|
||||
"з": "з", # Cyrillic 'з'
|
||||
"д": "д", # Cyrillic 'д'
|
||||
"б": "б", # Cyrillic 'б'
|
||||
"л": "л", # Cyrillic 'л'
|
||||
"я": "я", # Cyrillic 'я'
|
||||
"н": "н", # Already mapped, but explicit
|
||||
# Special characters
|
||||
"@": "а",
|
||||
# Multi-character visual bypasses (handled separately in preprocessing)
|
||||
# "}{" visually forms "х" - handled in _preprocess_visual_bypasses
|
||||
}
|
||||
|
||||
_RAW_PHRASE_GROUPS: Tuple[Tuple[str, Tuple[str, ...]], ...] = (
|
||||
("generic", ("айфон", "топ")),
|
||||
("generic", ("самсунг", "говно")),
|
||||
)
|
||||
|
||||
_SENSITIVE_PHRASE_PATH = Path("data/profanity/sensitive_phrases.json")
|
||||
_PHRASE_CACHE: dict[str, Tuple[Tuple[str, ...], ...]] = {}
|
||||
|
||||
|
||||
def _preprocess_visual_bypasses(text: str) -> str:
|
||||
"""
|
||||
Preprocess text to convert multi-character visual bypasses to their intended letters.
|
||||
This handles cases like "}{" visually forming "х".
|
||||
"""
|
||||
result = text
|
||||
# Convert "}{" to "х" (visual bypass for Cyrillic х)
|
||||
# The curly braces visually form the letter х when placed together
|
||||
result = result.replace("}{", "х")
|
||||
return result
|
||||
|
||||
|
||||
def _normalize_char(ch: str) -> str:
|
||||
"""Normalize a single character, mapping homoglyphs to canonical form."""
|
||||
# First try direct mapping (preserves case for non-mapped chars)
|
||||
if ch in _LEET_MAP:
|
||||
return _LEET_MAP[ch]
|
||||
# Then try lowercase mapping
|
||||
lower = ch.lower()
|
||||
if lower in _LEET_MAP:
|
||||
return _LEET_MAP[lower]
|
||||
# If no mapping and character is ASCII letter, return lowercase
|
||||
# This preserves English words like "fromchat" as-is
|
||||
if ch.isascii() and ch.isalpha():
|
||||
return lower
|
||||
# For other characters, return lowercase for consistency
|
||||
return lower
|
||||
|
||||
|
||||
def _normalize_token(token: str) -> str:
|
||||
"""Normalize a token by mapping all homoglyphs."""
|
||||
return "".join(_normalize_char(ch) for ch in token)
|
||||
|
||||
|
||||
def _normalize_text_for_profanity(text: str) -> str:
|
||||
"""
|
||||
Normalize entire text by mapping homoglyphs to canonical forms.
|
||||
This prevents bypasses like using English 'u' instead of Russian 'у'.
|
||||
"""
|
||||
return "".join(_normalize_char(ch) for ch in text)
|
||||
|
||||
|
||||
def _strip_zero_width_chars(text: str) -> str:
|
||||
"""
|
||||
Remove zero-width characters that could be used to bypass filters.
|
||||
"""
|
||||
# Zero-width space, zero-width non-joiner, zero-width joiner, etc.
|
||||
zero_width_chars = [
|
||||
'\u200B', # Zero-width space
|
||||
'\u200C', # Zero-width non-joiner
|
||||
'\u200D', # Zero-width joiner
|
||||
'\uFEFF', # Zero-width no-break space
|
||||
'\u2060', # Word joiner
|
||||
'\u2061', # Function application
|
||||
'\u2062', # Invisible times
|
||||
'\u2063', # Invisible separator
|
||||
'\u2064', # Invisible plus
|
||||
]
|
||||
result = text
|
||||
for zw_char in zero_width_chars:
|
||||
result = result.replace(zw_char, '')
|
||||
return result
|
||||
|
||||
|
||||
def _extract_alphanumeric_with_mapping(text: str, preserve_spaces: bool = False) -> tuple[str, list[int]]:
|
||||
"""
|
||||
Extract only alphanumeric characters from text and create a mapping
|
||||
from normalized positions to original positions.
|
||||
|
||||
Args:
|
||||
preserve_spaces: If True, preserve spaces in the normalized text (for phrase matching)
|
||||
|
||||
Returns:
|
||||
(normalized_text, position_map) where position_map[i] is the original
|
||||
position of the i-th character in normalized_text
|
||||
"""
|
||||
# First preprocess visual bypasses (like "}{" -> "х")
|
||||
text = _preprocess_visual_bypasses(text)
|
||||
|
||||
# Then normalize Unicode (composed vs decomposed)
|
||||
normalized_unicode = unicodedata.normalize('NFKC', text)
|
||||
|
||||
# For phrase matching, convert zero-width chars to spaces instead of stripping
|
||||
if preserve_spaces:
|
||||
zero_width_chars = ['\u200B', '\u200C', '\u200D', '\uFEFF', '\u2060', '\u2061', '\u2062', '\u2063', '\u2064']
|
||||
for zw_char in zero_width_chars:
|
||||
normalized_unicode = normalized_unicode.replace(zw_char, ' ')
|
||||
else:
|
||||
# Strip zero-width characters
|
||||
normalized_unicode = _strip_zero_width_chars(normalized_unicode)
|
||||
|
||||
normalized = []
|
||||
position_map = []
|
||||
|
||||
for i, ch in enumerate(normalized_unicode):
|
||||
# Check if character is alphanumeric (including Cyrillic)
|
||||
if ch.isalnum():
|
||||
# For phrase matching, preserve ASCII letters as-is (just lowercase)
|
||||
# to allow English words in patterns to match
|
||||
if preserve_spaces and ch.isascii() and ch.isalpha():
|
||||
normalized.append(ch.lower())
|
||||
else:
|
||||
# Normalize this character (homoglyphs, Cyrillic, etc.)
|
||||
normalized.append(_normalize_char(ch))
|
||||
position_map.append(i)
|
||||
elif preserve_spaces:
|
||||
# For phrase matching, treat any whitespace or non-alphanumeric as word separator
|
||||
if ch.isspace() or not ch.isalnum():
|
||||
# Normalize to single space to allow patterns to match
|
||||
if normalized and normalized[-1] != ' ': # Don't add consecutive spaces
|
||||
normalized.append(' ')
|
||||
position_map.append(i)
|
||||
|
||||
return "".join(normalized), position_map
|
||||
|
||||
|
||||
def _check_profanity_substrings(normalized_text: str, profane_words: Set[str]) -> list[tuple[int, int]]:
|
||||
"""
|
||||
Check for profane words as substrings or subsequences in normalized text.
|
||||
This catches cases like "хуй" in "хууй" (with extra characters).
|
||||
Returns list of (start, end) positions where profanity is found.
|
||||
"""
|
||||
spans = []
|
||||
normalized_lower = normalized_text.lower()
|
||||
|
||||
for word in profane_words:
|
||||
word_lower = word.lower()
|
||||
|
||||
# First try exact substring match
|
||||
start = 0
|
||||
while True:
|
||||
pos = normalized_lower.find(word_lower, start)
|
||||
if pos == -1:
|
||||
break
|
||||
spans.append((pos, pos + len(word_lower)))
|
||||
start = pos + 1
|
||||
|
||||
# Also check if profane word appears as a subsequence (allowing extra chars)
|
||||
# This catches cases like "хуй" in "хууй" or "х}{¥€уй" -> "хууй"
|
||||
# Now applies to ALL words, not just length >= 4, to prevent bypasses
|
||||
word_chars = list(word_lower)
|
||||
text_chars = list(normalized_lower)
|
||||
|
||||
# Stricter span limits based on word length to prevent false positives
|
||||
# Shorter words get much stricter limits
|
||||
if len(word_lower) <= 3:
|
||||
max_span_ratio = 1.3 # Very strict for 3-char words (e.g., "хуй")
|
||||
elif len(word_lower) == 4:
|
||||
max_span_ratio = 1.4 # Strict for 4-char words
|
||||
elif len(word_lower) <= 5:
|
||||
max_span_ratio = 1.5 # Moderate for 5-char words
|
||||
else:
|
||||
max_span_ratio = 1.8 # Slightly more lenient for longer words
|
||||
|
||||
# Try to find the word as a subsequence
|
||||
i = 0 # position in text
|
||||
j = 0 # position in word
|
||||
seq_start = None
|
||||
|
||||
while i < len(text_chars) and j < len(word_chars):
|
||||
if text_chars[i] == word_chars[j]:
|
||||
if seq_start is None:
|
||||
seq_start = i
|
||||
j += 1
|
||||
if j == len(word_chars):
|
||||
# Found the word as subsequence
|
||||
seq_end = i + 1
|
||||
# Check if the span is reasonable (not too long)
|
||||
span_length = seq_end - seq_start
|
||||
max_allowed_span = int(len(word_lower) * max_span_ratio)
|
||||
if span_length <= max_allowed_span:
|
||||
# Only add if it's not already covered by exact match
|
||||
if (seq_start, seq_end) not in spans:
|
||||
spans.append((seq_start, seq_end))
|
||||
# Reset to find next occurrence - continue from after the end of this match
|
||||
next_start = seq_start + 1
|
||||
seq_start = None
|
||||
j = 0
|
||||
i = next_start
|
||||
continue
|
||||
i += 1
|
||||
|
||||
return spans
|
||||
|
||||
|
||||
def _check_profanity_in_normalized(normalized_text: str) -> bool:
|
||||
"""
|
||||
Check if normalized text contains profanity.
|
||||
Uses both better_profanity library and substring matching for better detection.
|
||||
|
||||
Returns True if profanity is found.
|
||||
"""
|
||||
if not normalized_text:
|
||||
return False
|
||||
|
||||
# Check normalized text for profanity using better_profanity
|
||||
censored = _profanity.censor(normalized_text, censor_char="\\*")
|
||||
|
||||
# Check if better_profanity found anything
|
||||
if "*" in censored:
|
||||
return True
|
||||
|
||||
# Also check for profane words as substrings (to catch cases like "хуй" in "хууй" or "хуйня")
|
||||
profane_words = _STATIC_TERMS
|
||||
substring_spans = _check_profanity_substrings(normalized_text, profane_words)
|
||||
|
||||
# If we found any substring matches, there's profanity
|
||||
if substring_spans:
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
|
||||
def _tokenize_with_spans(text: str) -> List[Tuple[int, int, str]]:
|
||||
tokens: List[Tuple[int, int, str]] = []
|
||||
start: int | None = None
|
||||
buffer: List[str] = []
|
||||
|
||||
for idx, ch in enumerate(text):
|
||||
if ch.isalnum() or ch in {"@", "#", "_"}:
|
||||
if start is None:
|
||||
start = idx
|
||||
buffer.append(ch)
|
||||
else:
|
||||
if buffer and start is not None:
|
||||
token_raw = "".join(buffer)
|
||||
tokens.append((start, idx, _normalize_token(token_raw)))
|
||||
buffer.clear()
|
||||
start = None
|
||||
if buffer and start is not None:
|
||||
token_raw = "".join(buffer)
|
||||
tokens.append((start, len(text), _normalize_token(token_raw)))
|
||||
return tokens
|
||||
|
||||
|
||||
def _edit_distance_limited(a: str, b: str, max_distance: int = 1) -> bool:
|
||||
if a == b:
|
||||
return True
|
||||
if max_distance <= 0:
|
||||
return False
|
||||
if abs(len(a) - len(b)) > max_distance:
|
||||
return False
|
||||
|
||||
previous = list(range(len(b) + 1))
|
||||
for i, ca in enumerate(a, 1):
|
||||
current = [i]
|
||||
best = current[0]
|
||||
for j, cb in enumerate(b, 1):
|
||||
insert_cost = current[j - 1] + 1
|
||||
delete_cost = previous[j] + 1
|
||||
replace_cost = previous[j - 1] + (0 if ca == cb else 1)
|
||||
cost = min(insert_cost, delete_cost, replace_cost)
|
||||
current.append(cost)
|
||||
if cost < best:
|
||||
best = cost
|
||||
if best > max_distance:
|
||||
return False
|
||||
previous = current
|
||||
return previous[-1] <= max_distance
|
||||
|
||||
|
||||
def _load_sensitive_phrases() -> List[Tuple[str, ...]]:
|
||||
if not _SENSITIVE_PHRASE_PATH.exists():
|
||||
return []
|
||||
try:
|
||||
payload = json.loads(_SENSITIVE_PHRASE_PATH.read_text(encoding="utf-8"))
|
||||
phrases: List[Tuple[str, ...]] = []
|
||||
if isinstance(payload, list):
|
||||
for entry in payload:
|
||||
if isinstance(entry, list) and entry:
|
||||
normalized = tuple(str(part).strip() for part in entry if str(part).strip())
|
||||
if normalized:
|
||||
phrases.append(normalized)
|
||||
return phrases
|
||||
except Exception:
|
||||
return []
|
||||
|
||||
|
||||
def _get_phrases(group: str) -> Tuple[Tuple[str, ...], ...]:
|
||||
if group not in _PHRASE_CACHE:
|
||||
base = [phrase for key, phrase in _RAW_PHRASE_GROUPS if key == group]
|
||||
if group == "sensitive":
|
||||
base.extend(_load_sensitive_phrases())
|
||||
_PHRASE_CACHE[group] = tuple(
|
||||
tuple(_normalize_token(part) for part in phrase)
|
||||
for phrase in base
|
||||
)
|
||||
return _PHRASE_CACHE[group]
|
||||
|
||||
|
||||
def _find_fuzzy_phrase_spans(text: str, group: str = "generic") -> List[Tuple[int, int]]:
|
||||
tokens = _tokenize_with_spans(text)
|
||||
if not tokens:
|
||||
return []
|
||||
|
||||
spans: List[Tuple[int, int]] = []
|
||||
normalized_phrases = _get_phrases(group)
|
||||
|
||||
for index in range(len(tokens)):
|
||||
for phrase in normalized_phrases:
|
||||
if index + len(phrase) > len(tokens):
|
||||
continue
|
||||
matches = True
|
||||
for offset, target in enumerate(phrase):
|
||||
token = tokens[index + offset][2]
|
||||
if not _edit_distance_limited(token, target):
|
||||
matches = False
|
||||
break
|
||||
if matches:
|
||||
span_start = tokens[index][0]
|
||||
span_end = tokens[index + len(phrase) - 1][1]
|
||||
spans.append((span_start, span_end))
|
||||
return spans
|
||||
|
||||
_dictionary_lock = RLock()
|
||||
_blocklist_signature: Tuple[str, ...] | None = None
|
||||
_profanity = Profanity()
|
||||
|
||||
|
||||
def _normalize_words(words: Iterable[str]) -> Set[str]:
|
||||
normalized: Set[str] = set()
|
||||
for raw in words:
|
||||
if not raw:
|
||||
continue
|
||||
cleaned = re.sub(r"\s+", " ", str(raw)).strip().lower()
|
||||
if cleaned:
|
||||
normalized.add(cleaned)
|
||||
return normalized
|
||||
|
||||
|
||||
def _load_blocklist() -> Set[str]:
|
||||
if not BLOCKLIST_PATH.exists():
|
||||
return set()
|
||||
try:
|
||||
data = json.loads(BLOCKLIST_PATH.read_text(encoding="utf-8"))
|
||||
if isinstance(data, list):
|
||||
return _normalize_words(data)
|
||||
except Exception:
|
||||
pass
|
||||
return set()
|
||||
|
||||
|
||||
def _write_blocklist(words: Iterable[str]) -> None:
|
||||
BLOCKLIST_PATH.write_text(
|
||||
json.dumps(sorted(words), ensure_ascii=False, indent=2) + "\n",
|
||||
encoding="utf-8"
|
||||
)
|
||||
|
||||
|
||||
def _rebuild_dictionary(force: bool = False) -> None:
|
||||
global _profanity, _blocklist_signature
|
||||
with _dictionary_lock:
|
||||
blocklist_list = sorted(_load_blocklist())
|
||||
signature = tuple(blocklist_list)
|
||||
if not force and _blocklist_signature == signature and _blocklist_signature is not None:
|
||||
return
|
||||
|
||||
profanity = Profanity()
|
||||
profanity.load_censor_words()
|
||||
# Remove whitelisted words from the default word list
|
||||
try:
|
||||
for word in _WHITELIST:
|
||||
profanity.remove_censor_words([word])
|
||||
except AttributeError:
|
||||
# If remove_censor_words doesn't exist, we'll handle it in post-processing
|
||||
pass
|
||||
combined = set(_STATIC_TERMS)
|
||||
combined.update(blocklist_list)
|
||||
# Remove whitelisted words from our custom terms
|
||||
combined -= _WHITELIST
|
||||
if combined:
|
||||
profanity.add_censor_words(list(combined))
|
||||
|
||||
_profanity = profanity
|
||||
_blocklist_signature = signature
|
||||
|
||||
|
||||
def _check_phrase_patterns(text: str) -> bool:
|
||||
"""
|
||||
Check if text matches any phrase patterns.
|
||||
Returns True if any pattern matches.
|
||||
"""
|
||||
# Normalize text for phrase matching (remove special chars but preserve spaces)
|
||||
normalized_text, _ = _extract_alphanumeric_with_mapping(text, preserve_spaces=True)
|
||||
normalized_lower = normalized_text.lower()
|
||||
|
||||
# Check phrase patterns
|
||||
for pattern in _PHRASE_PATTERNS:
|
||||
if pattern.search(normalized_lower):
|
||||
return True
|
||||
|
||||
# Check fuzzy phrase spans
|
||||
if _find_fuzzy_phrase_spans(normalized_lower, "generic"):
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
|
||||
def contains_profanity(text: str) -> bool:
|
||||
"""
|
||||
Check if text contains profanity.
|
||||
Returns True if profanity is detected.
|
||||
"""
|
||||
if not text:
|
||||
return False
|
||||
|
||||
_rebuild_dictionary()
|
||||
|
||||
# Check original text patterns first (before normalization) to catch visual bypasses
|
||||
# like "}{" used to form "х"
|
||||
for pattern in _ORIGINAL_TEXT_PATTERNS:
|
||||
if pattern.search(text):
|
||||
return True
|
||||
|
||||
# Check phrase patterns
|
||||
if _check_phrase_patterns(text):
|
||||
return True
|
||||
|
||||
# Normalize text for whitelist matching (to handle special characters)
|
||||
normalized_for_whitelist, _ = _extract_alphanumeric_with_mapping(text)
|
||||
normalized_for_whitelist_lower = normalized_for_whitelist.lower()
|
||||
|
||||
# Check if text contains whitelisted words - if the entire text is a whitelisted word, skip profanity check
|
||||
for whitelist_word in _WHITELIST:
|
||||
normalized_whitelist, _ = _extract_alphanumeric_with_mapping(whitelist_word)
|
||||
normalized_whitelist_lower = normalized_whitelist.lower()
|
||||
|
||||
# Check if the normalized text exactly matches a whitelisted word
|
||||
if normalized_for_whitelist_lower == normalized_whitelist_lower:
|
||||
return False
|
||||
|
||||
# Extract only alphanumeric characters and normalize homoglyphs
|
||||
# This removes special characters, emojis, etc. that could be used to bypass the filter
|
||||
normalized_text, _ = _extract_alphanumeric_with_mapping(text)
|
||||
|
||||
# Check profanity on normalized text (without special characters)
|
||||
return _check_profanity_in_normalized(normalized_text)
|
||||
|
||||
|
||||
def contains_sensitive_phrase(text: str) -> bool:
|
||||
if not text:
|
||||
return False
|
||||
if _find_fuzzy_phrase_spans(text, "sensitive"):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def get_blocklist() -> List[str]:
|
||||
with _dictionary_lock:
|
||||
return sorted(_load_blocklist())
|
||||
|
||||
|
||||
def add_to_blocklist(words: Iterable[str]) -> Tuple[List[str], List[str]]:
|
||||
normalized = _normalize_words(words)
|
||||
if not normalized:
|
||||
return [], get_blocklist()
|
||||
|
||||
with _dictionary_lock:
|
||||
current = _load_blocklist()
|
||||
added = sorted(normalized - current)
|
||||
if not added:
|
||||
return [], sorted(current)
|
||||
|
||||
updated = sorted(current | normalized)
|
||||
_write_blocklist(updated)
|
||||
_rebuild_dictionary(force=True)
|
||||
return added, updated
|
||||
|
||||
|
||||
def remove_from_blocklist(words: Iterable[str]) -> Tuple[List[str], List[str]]:
|
||||
normalized = _normalize_words(words)
|
||||
if not normalized:
|
||||
return [], get_blocklist()
|
||||
|
||||
with _dictionary_lock:
|
||||
current = _load_blocklist()
|
||||
removed = sorted(word for word in normalized if word in current)
|
||||
if not removed:
|
||||
return [], sorted(current)
|
||||
|
||||
updated = sorted(current - normalized)
|
||||
_write_blocklist(updated)
|
||||
_rebuild_dictionary(force=True)
|
||||
return removed, updated
|
||||
|
||||
@@ -0,0 +1,232 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import time
|
||||
from typing import Callable
|
||||
from fastapi import Request
|
||||
from slowapi import Limiter
|
||||
from slowapi.util import get_remote_address
|
||||
|
||||
from ..utils import get_client_ip
|
||||
|
||||
logger = logging.getLogger("uvicorn.error")
|
||||
|
||||
def get_ip_key(request: Request) -> str:
|
||||
"""Get rate limit key based on IP address."""
|
||||
return get_client_ip(request) or get_remote_address(request)
|
||||
|
||||
# Initialize limiter with IP-based key function
|
||||
# Note: We don't set default_limits to avoid affecting all users if one IP is attacked.
|
||||
# Each endpoint should have an explicit rate limit based on its sensitivity.
|
||||
# Rate limits automatically expire after the time window - IPs are not permanently blocked.
|
||||
limiter = Limiter(
|
||||
key_func=get_ip_key,
|
||||
default_limits=[], # No global default - each endpoint must have explicit limits
|
||||
storage_uri="memory://", # In-memory storage (can be changed to Redis later)
|
||||
)
|
||||
|
||||
|
||||
# Rate limit decorator for IP-based limiting
|
||||
def rate_limit_per_ip(limit: str) -> Callable:
|
||||
"""Rate limit based on IP address."""
|
||||
return limiter.limit(limit, key_func=get_ip_key)
|
||||
|
||||
|
||||
def _get_storage_dict(storage) -> dict | None:
|
||||
"""Get the internal storage dictionary from slowapi's memory storage."""
|
||||
if hasattr(storage, "_storage") and isinstance(storage._storage, dict):
|
||||
return storage._storage
|
||||
elif hasattr(storage, "storage") and isinstance(storage.storage, dict):
|
||||
return storage.storage
|
||||
return None
|
||||
|
||||
|
||||
def reset_all_rate_limits() -> int:
|
||||
"""
|
||||
Reset all rate limits by clearing the storage.
|
||||
This should be called on startup to ensure a clean state.
|
||||
Returns the number of entries cleared.
|
||||
"""
|
||||
try:
|
||||
# Access the private _storage attribute
|
||||
storage = limiter._storage
|
||||
storage_dict = _get_storage_dict(storage)
|
||||
|
||||
if storage_dict is None:
|
||||
# Try using the storage's reset method if available
|
||||
if hasattr(storage, "reset"):
|
||||
try:
|
||||
# Try reset() with no args first (clears all)
|
||||
storage.reset()
|
||||
logger.info("Reset all rate limits on startup using storage.reset()")
|
||||
return 1 # Assume it worked
|
||||
except TypeError:
|
||||
# reset() might require arguments, try clearing differently
|
||||
try:
|
||||
# Some storage backends need explicit clearing
|
||||
if hasattr(storage, "clear"):
|
||||
storage.clear()
|
||||
logger.info("Reset all rate limits on startup using storage.clear()")
|
||||
return 1
|
||||
except Exception:
|
||||
pass
|
||||
except Exception:
|
||||
pass
|
||||
logger.warning("Could not reset rate limits: storage dict not accessible and no reset method")
|
||||
return 0
|
||||
|
||||
count = len(storage_dict)
|
||||
if count > 0:
|
||||
storage_dict.clear()
|
||||
logger.info(f"Reset all rate limits on startup: cleared {count} entries")
|
||||
return count
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to reset rate limits on startup: {e}")
|
||||
return 0
|
||||
|
||||
|
||||
def reset_rate_limit_for_ip(ip: str) -> bool:
|
||||
"""
|
||||
Manually reset rate limit for a specific IP address.
|
||||
This clears all rate limit entries for the given IP.
|
||||
Returns True if any entries were cleared, False otherwise.
|
||||
"""
|
||||
if not ip:
|
||||
return False
|
||||
|
||||
try:
|
||||
# Access the private _storage attribute
|
||||
storage = limiter._storage
|
||||
storage_dict = _get_storage_dict(storage)
|
||||
|
||||
if storage_dict is None:
|
||||
# Try alternative methods
|
||||
if hasattr(storage, "reset"):
|
||||
try:
|
||||
storage.reset(ip)
|
||||
return True
|
||||
except Exception:
|
||||
pass
|
||||
return False
|
||||
|
||||
cleared = False
|
||||
# slowapi stores entries with keys like "LIMITER:{ip}:{endpoint}"
|
||||
# We need to find all keys that contain this IP
|
||||
# Also handle cases where IP might be in different positions
|
||||
keys_to_remove = []
|
||||
|
||||
for key in list(storage_dict.keys()):
|
||||
if isinstance(key, str):
|
||||
# Check multiple patterns:
|
||||
# - "LIMITER:{ip}:{endpoint}"
|
||||
# - Keys containing the IP anywhere
|
||||
# - Keys starting with the IP
|
||||
if (key.startswith(f"LIMITER:{ip}:") or
|
||||
key.startswith(f"LIMITER:{ip}") or
|
||||
f":{ip}:" in key or
|
||||
key.endswith(f":{ip}") or
|
||||
(ip in key and "LIMITER" in key)):
|
||||
keys_to_remove.append(key)
|
||||
|
||||
for key in keys_to_remove:
|
||||
try:
|
||||
del storage_dict[key]
|
||||
cleared = True
|
||||
logger.info(f"Cleared rate limit key: {key}")
|
||||
except KeyError:
|
||||
pass
|
||||
|
||||
if cleared:
|
||||
logger.info(f"Successfully cleared rate limits for IP: {ip}")
|
||||
else:
|
||||
logger.warning(f"No rate limit entries found for IP: {ip}")
|
||||
|
||||
return cleared
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to reset rate limit for IP {ip}: {e}")
|
||||
return False
|
||||
|
||||
|
||||
def clear_all_rate_limits() -> int:
|
||||
"""
|
||||
Clear all rate limit entries. Use with caution - this affects all IPs.
|
||||
Returns the number of entries cleared.
|
||||
"""
|
||||
try:
|
||||
# Access the private _storage attribute
|
||||
storage = limiter._storage
|
||||
storage_dict = _get_storage_dict(storage)
|
||||
|
||||
if storage_dict is None:
|
||||
return 0
|
||||
|
||||
count = len(storage_dict)
|
||||
storage_dict.clear()
|
||||
logger.warning(f"Cleared all {count} rate limit entries")
|
||||
return count
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to clear all rate limits: {e}")
|
||||
return 0
|
||||
|
||||
|
||||
def cleanup_expired_rate_limits() -> int:
|
||||
"""
|
||||
Clean up expired rate limit entries from memory storage.
|
||||
This helps prevent rate limits from being stuck indefinitely.
|
||||
Returns the number of entries cleaned up.
|
||||
"""
|
||||
try:
|
||||
# Access the private _storage attribute
|
||||
storage = limiter._storage
|
||||
storage_dict = _get_storage_dict(storage)
|
||||
|
||||
if storage_dict is None:
|
||||
return 0
|
||||
|
||||
# slowapi's memory storage stores entries as tuples: (count, reset_time)
|
||||
# Entries should expire naturally, but we'll clean up any that are clearly expired
|
||||
now = time.time()
|
||||
cleaned = 0
|
||||
keys_to_remove = []
|
||||
|
||||
for key, value in storage_dict.items():
|
||||
if isinstance(value, (tuple, list)) and len(value) >= 2:
|
||||
# Check if reset_time has passed (with some buffer)
|
||||
reset_time = value[1] if isinstance(value[1], (int, float)) else 0
|
||||
# Add 60 second buffer to ensure we don't remove active entries
|
||||
if reset_time > 0 and now > (reset_time + 60):
|
||||
keys_to_remove.append(key)
|
||||
elif isinstance(value, dict):
|
||||
# Some storage formats use dicts with 'expiry' or 'reset' fields
|
||||
expiry = value.get("expiry") or value.get("reset") or value.get("reset_time")
|
||||
if expiry and isinstance(expiry, (int, float)) and now > (expiry + 60):
|
||||
keys_to_remove.append(key)
|
||||
|
||||
for key in keys_to_remove:
|
||||
try:
|
||||
del storage_dict[key]
|
||||
cleaned += 1
|
||||
except KeyError:
|
||||
pass
|
||||
|
||||
if cleaned > 0:
|
||||
logger.info(f"Cleaned up {cleaned} expired rate limit entries")
|
||||
|
||||
return cleaned
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to cleanup expired rate limits: {e}")
|
||||
return 0
|
||||
|
||||
|
||||
async def start_rate_limit_cleanup_task() -> None:
|
||||
"""Start a background task to periodically clean up expired rate limit entries."""
|
||||
while True:
|
||||
try:
|
||||
await asyncio.sleep(300) # Run every 5 minutes
|
||||
cleanup_expired_rate_limits()
|
||||
except asyncio.CancelledError:
|
||||
break
|
||||
except Exception as e:
|
||||
logger.error(f"Error in rate limit cleanup task: {e}")
|
||||
await asyncio.sleep(60) # Wait 1 minute before retrying
|
||||
Reference in New Issue
Block a user