mirror of
https://github.com/fromchat-messenger/web.git
synced 2026-09-22 19:15:08 +03:00
Remove user agent checks
This commit is contained in:
@@ -189,34 +189,6 @@ def _render_security(action: str, fields: Dict[str, Any]) -> List[str]:
|
||||
total = len(fields.get("words") or [])
|
||||
lines.append(f"Total entries: {total}")
|
||||
return lines
|
||||
if action == "blocked_user_agent":
|
||||
action_type = fields.get("action_type", "access")
|
||||
lines = [f"Blocked user agent attempted {action_type}"]
|
||||
if fields.get("username"):
|
||||
lines.append(f"Username: {fields['username']}")
|
||||
if fields.get("user_agent"):
|
||||
lines.append(f"User agent: {fields['user_agent']}")
|
||||
if fields.get("ip"):
|
||||
ip_raw = fields["ip"]
|
||||
ip_display = "localhost" if ip_raw in {"127.0.0.1", "::1"} else ip_raw
|
||||
lines.append(f"IP: {ip_display}")
|
||||
return lines
|
||||
if action == "user_agent_blocklist_add":
|
||||
added = fields.get("added") or []
|
||||
lines = [f"User agent blocklist updated by {_format_actor(fields, 'actor')}"]
|
||||
if added:
|
||||
lines.append(f"Added patterns: {', '.join(added)}")
|
||||
total = len(fields.get("patterns") or [])
|
||||
lines.append(f"Total patterns: {total}")
|
||||
return lines
|
||||
if action == "user_agent_blocklist_remove":
|
||||
removed = fields.get("removed") or []
|
||||
lines = [f"User agent blocklist cleaned by {_format_actor(fields, 'actor')}"]
|
||||
if removed:
|
||||
lines.append(f"Removed patterns: {', '.join(removed)}")
|
||||
total = len(fields.get("patterns") or [])
|
||||
lines.append(f"Total patterns: {total}")
|
||||
return lines
|
||||
return [f"{action.replace('_', ' ').capitalize()}"] + [
|
||||
f"{key.replace('_', ' ').capitalize()}: {value}"
|
||||
for key, value in fields.items()
|
||||
|
||||
@@ -1,245 +0,0 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
from pathlib import Path
|
||||
from threading import RLock
|
||||
from typing import Iterable, List, Set
|
||||
from user_agents import parse as parse_ua
|
||||
|
||||
BLOCKLIST_PATH = Path("data/user_agent_blocklist.json")
|
||||
BLOCKLIST_PATH.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Hardcoded list of known bot/scraper user agents
|
||||
_STATIC_BLOCKED_AGENTS: Set[str] = {
|
||||
"python-requests",
|
||||
"python requests",
|
||||
"requests",
|
||||
"curl",
|
||||
"wget",
|
||||
"httpie",
|
||||
"go-http-client",
|
||||
"java/",
|
||||
"okhttp",
|
||||
"apache-httpclient",
|
||||
"scrapy",
|
||||
"mechanize",
|
||||
"beautifulsoup",
|
||||
"urllib",
|
||||
"httpx",
|
||||
"aiohttp",
|
||||
"postman",
|
||||
"insomnia",
|
||||
"postmanruntime",
|
||||
"restclient",
|
||||
"http",
|
||||
"bot",
|
||||
"crawler",
|
||||
"spider",
|
||||
"scraper",
|
||||
}
|
||||
|
||||
_blocklist_lock = RLock()
|
||||
_blocklist_cache: Set[str] | None = None
|
||||
|
||||
|
||||
def _normalize_pattern(pattern: str) -> str:
|
||||
cleaned = re.sub(r"\s+", " ", str(pattern)).strip().lower()
|
||||
return cleaned
|
||||
|
||||
|
||||
def _load_blocklist() -> Set[str]:
|
||||
global _blocklist_cache
|
||||
with _blocklist_lock:
|
||||
if _blocklist_cache is not None:
|
||||
return _blocklist_cache
|
||||
|
||||
# Start with static hardcoded patterns
|
||||
patterns = set(_normalize_pattern(p) for p in _STATIC_BLOCKED_AGENTS)
|
||||
|
||||
# Load additional patterns from external file
|
||||
if BLOCKLIST_PATH.exists():
|
||||
try:
|
||||
data = json.loads(BLOCKLIST_PATH.read_text(encoding="utf-8"))
|
||||
if isinstance(data, list):
|
||||
external_patterns = set(_normalize_pattern(p) for p in data if p)
|
||||
patterns.update(external_patterns)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
_blocklist_cache = patterns
|
||||
return patterns
|
||||
|
||||
|
||||
def _write_blocklist(external_patterns: Iterable[str]) -> None:
|
||||
"""Write only external patterns to the JSON file. Static patterns are not stored."""
|
||||
normalized = sorted(set(_normalize_pattern(p) for p in external_patterns if p))
|
||||
BLOCKLIST_PATH.write_text(
|
||||
json.dumps(normalized, ensure_ascii=False, indent=2) + "\n",
|
||||
encoding="utf-8"
|
||||
)
|
||||
# Clear cache so it reloads with static + external patterns
|
||||
global _blocklist_cache
|
||||
with _blocklist_lock:
|
||||
_blocklist_cache = None
|
||||
|
||||
|
||||
def _match_pattern(text: str, pattern: str) -> bool:
|
||||
normalized_text = text.lower()
|
||||
normalized_pattern = pattern.lower()
|
||||
|
||||
if normalized_pattern in normalized_text:
|
||||
return True
|
||||
|
||||
try:
|
||||
regex = re.compile(normalized_pattern, re.IGNORECASE)
|
||||
if regex.search(normalized_text):
|
||||
return True
|
||||
except re.error:
|
||||
pass
|
||||
|
||||
return False
|
||||
|
||||
|
||||
def is_user_agent_blocked(raw_user_agent: str | None) -> bool:
|
||||
if not raw_user_agent:
|
||||
return False
|
||||
|
||||
blocklist = _load_blocklist()
|
||||
if not blocklist:
|
||||
return False
|
||||
|
||||
for pattern in blocklist:
|
||||
if _match_pattern(raw_user_agent, pattern):
|
||||
return True
|
||||
|
||||
try:
|
||||
ua = parse_ua(raw_user_agent)
|
||||
browser_name = ua.browser.family or ""
|
||||
os_name = ua.os.family or ""
|
||||
|
||||
browser_pattern = browser_name.lower() if browser_name else ""
|
||||
os_pattern = os_name.lower() if os_name else ""
|
||||
|
||||
formatted = f"{os_name or 'Other'}, {browser_name or 'Unknown browser'}"
|
||||
if ua.browser.version_string:
|
||||
formatted = f"{formatted} {ua.browser.version_string}"
|
||||
|
||||
for pattern in blocklist:
|
||||
if _match_pattern(formatted, pattern):
|
||||
return True
|
||||
if browser_pattern and _match_pattern(browser_pattern, pattern):
|
||||
return True
|
||||
if os_pattern and _match_pattern(os_pattern, pattern):
|
||||
return True
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
return False
|
||||
|
||||
|
||||
def get_blocklist() -> List[str]:
|
||||
"""Get all blocked patterns (static + external)."""
|
||||
with _blocklist_lock:
|
||||
return sorted(_load_blocklist())
|
||||
|
||||
|
||||
def get_static_blocklist() -> List[str]:
|
||||
"""Get only the hardcoded static patterns."""
|
||||
return sorted(_normalize_pattern(p) for p in _STATIC_BLOCKED_AGENTS)
|
||||
|
||||
|
||||
def get_external_blocklist() -> List[str]:
|
||||
"""Get only the patterns from the external JSON file."""
|
||||
if not BLOCKLIST_PATH.exists():
|
||||
return []
|
||||
try:
|
||||
data = json.loads(BLOCKLIST_PATH.read_text(encoding="utf-8"))
|
||||
if isinstance(data, list):
|
||||
return sorted(_normalize_pattern(p) for p in data if p)
|
||||
except Exception:
|
||||
pass
|
||||
return []
|
||||
|
||||
|
||||
def add_to_blocklist(patterns: Iterable[str]) -> tuple[List[str], List[str]]:
|
||||
"""Add patterns to the external blocklist. Static patterns cannot be modified."""
|
||||
normalized = set(_normalize_pattern(p) for p in patterns if p)
|
||||
if not normalized:
|
||||
return [], get_blocklist()
|
||||
|
||||
with _blocklist_lock:
|
||||
# Only add to external blocklist, not static
|
||||
static_patterns = set(_normalize_pattern(p) for p in _STATIC_BLOCKED_AGENTS)
|
||||
|
||||
# Filter out static patterns (they're already blocked)
|
||||
normalized = normalized - static_patterns
|
||||
if not normalized:
|
||||
return [], get_blocklist()
|
||||
|
||||
# Load current external patterns
|
||||
external_current = set()
|
||||
if BLOCKLIST_PATH.exists():
|
||||
try:
|
||||
data = json.loads(BLOCKLIST_PATH.read_text(encoding="utf-8"))
|
||||
if isinstance(data, list):
|
||||
external_current = set(_normalize_pattern(p) for p in data if p)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
added = sorted(normalized - external_current)
|
||||
if not added:
|
||||
return [], get_blocklist()
|
||||
|
||||
updated_external = sorted(external_current | normalized)
|
||||
_write_blocklist(updated_external)
|
||||
|
||||
# Clear cache to reload
|
||||
_blocklist_cache = None
|
||||
|
||||
return added, get_blocklist()
|
||||
|
||||
|
||||
def remove_from_blocklist(patterns: Iterable[str]) -> tuple[List[str], List[str]]:
|
||||
"""Remove patterns from the external blocklist. Static patterns cannot be removed."""
|
||||
normalized = set(_normalize_pattern(p) for p in patterns if p)
|
||||
if not normalized:
|
||||
return [], get_blocklist()
|
||||
|
||||
with _blocklist_lock:
|
||||
# Only remove from external blocklist, not static
|
||||
static_patterns = set(_normalize_pattern(p) for p in _STATIC_BLOCKED_AGENTS)
|
||||
|
||||
# Filter out static patterns (cannot remove them)
|
||||
normalized = normalized - static_patterns
|
||||
if not normalized:
|
||||
return [], get_blocklist()
|
||||
|
||||
# Load current external patterns
|
||||
external_current = set()
|
||||
if BLOCKLIST_PATH.exists():
|
||||
try:
|
||||
data = json.loads(BLOCKLIST_PATH.read_text(encoding="utf-8"))
|
||||
if isinstance(data, list):
|
||||
external_current = set(_normalize_pattern(p) for p in data if p)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
removed = sorted(pattern for pattern in normalized if pattern in external_current)
|
||||
if not removed:
|
||||
return [], get_blocklist()
|
||||
|
||||
updated_external = sorted(external_current - normalized)
|
||||
_write_blocklist(updated_external)
|
||||
|
||||
# Clear cache to reload
|
||||
_blocklist_cache = None
|
||||
|
||||
return removed, get_blocklist()
|
||||
|
||||
|
||||
def clear_blocklist_cache() -> None:
|
||||
global _blocklist_cache
|
||||
with _blocklist_lock:
|
||||
_blocklist_cache = None
|
||||
|
||||
Reference in New Issue
Block a user