From 26555a6a4c5094187a1eb612323291b4efdb81cf Mon Sep 17 00:00:00 2001 From: denis0001-dev Date: Tue, 14 Jul 2026 17:53:07 +0300 Subject: [PATCH] Fix --- .env.example | 5 +- README.md | 26 +++--- compose.yml | 6 +- updater/git_remote.py | 180 ++++++++++++++++++++++++++++++--------- updater/github_client.py | 7 +- updater/main.py | 5 ++ 6 files changed, 172 insertions(+), 57 deletions(-) diff --git a/.env.example b/.env.example index bdd4b85..56d866d 100644 --- a/.env.example +++ b/.env.example @@ -1,11 +1,12 @@ -GITHUB_TOKEN= -GIT_TOKEN= +# Repo/config for the updater. UPDATER_TOKEN lives in deployment/.env.prod (copied to server .env). BACKEND_REPO=https://github.com/fromchat-messenger/backend.git WEB_REPO=https://github.com/fromchat-messenger/web.git DEPLOYMENT_REPO=https://github.com/fromchat-messenger/app.git COMPOSE_PROJECT_DIR=/home/user/fromchat-server FROMCHAT_COMPONENTS=backend,frontend,caddy CHECK_INTERVAL_SECONDS=60 +# Optional: force Gitea API base (otherwise host-gateway :3000 is probed first) +# GITEA_BASE=http://host.docker.internal:3000 # Gitea-only example (no GitHub fallback needed): # BACKEND_REPO=https://git.fromchat.ru/FromChat/backend.git diff --git a/README.md b/README.md index 59c6f56..2087ab1 100644 --- a/README.md +++ b/README.md @@ -4,22 +4,22 @@ Watches GitHub **tags** and **container packages** for backend + frontend repos. When a newer semver tag exists in both and all required `fromchat/*` images are published, regenerates `compose.yml` (via the deployment merge script) and restarts the stack. -## Environment (`.env`) +## Environment -| Variable | Description | -|----------|-------------| -| `GITHUB_TOKEN` | PAT with `read:packages` + `repo` | -| `BACKEND_REPO` | e.g. `https://github.com/fromchat-messenger/backend.git` | -| `WEB_REPO` | e.g. `https://github.com/fromchat-messenger/web.git` | -| `DEPLOYMENT_REPO` | `https://github.com/fromchat-messenger/app.git` (deployment scripts) | -| `DEPLOYMENT_REPO` | Deployment repo for `generate-compose.py` | -| `COMPOSE_PROJECT_DIR` | Host path to the stack (`compose.yml` parent directory) | -| `FROMCHAT_COMPONENTS` | Comma list: `backend,frontend,caddy,updater` | -| `CHECK_INTERVAL_SECONDS` | Poll interval (default `60`) | +Production stack: the updater service loads **both** the main stack `.env` (secrets from `deployment/.env.prod`) and `updater/.env` (repo URLs and paths below). -Only `compose.yml` from the stack directory is mounted into the updater container (read-write at `/fromchat/compose.yml`). The current release is read from image tags inside that file. +| Variable | Where | Description | +|----------|-------|-------------| +| `UPDATER_TOKEN` | main `.env` | PAT with `read:packages` + `repo` (GitHub or Gitea) | +| `BACKEND_REPO` | `updater/.env` | e.g. `https://github.com/fromchat-messenger/backend.git` | +| `WEB_REPO` | `updater/.env` | e.g. `https://github.com/fromchat-messenger/web.git` | +| `DEPLOYMENT_REPO` | `updater/.env` | Deployment repo for `generate-compose.py` | +| `COMPOSE_PROJECT_DIR` | `updater/.env` | Host path to the stack (`compose.yml` parent directory) | +| `FROMCHAT_COMPONENTS` | `updater/.env` | Comma list: `backend,frontend,caddy,updater` | +| `CHECK_INTERVAL_SECONDS` | `updater/.env` | Poll interval (default `60`) | +| `GITEA_BASE` | optional | Override Gitea API base (e.g. `http://host.docker.internal:3000`). If unset, probes host-gateway `:3000` then `https://git.fromchat.ru`. | -Create token: [GitHub PAT (read:packages + repo)](https://github.com/settings/tokens/new?description=FromChat%20Updater&scopes=read:packages,repo) +Create token: [GitHub PAT (read:packages + repo)](https://github.com/settings/tokens/new?description=FromChat%20Updater&scopes=read:packages,repo) — set as `UPDATER_TOKEN` in `deployment/.env.prod` (deploy copies it to the server main `.env`). ## Run standalone diff --git a/compose.yml b/compose.yml index 08da996..c09c56b 100644 --- a/compose.yml +++ b/compose.yml @@ -1,12 +1,16 @@ services: updater: - image: fromchat/updater:latest + build: + context: . + dockerfile: Dockerfile restart: unless-stopped env_file: - .env volumes: - ${COMPOSE_PROJECT_DIR}/compose.yml:/fromchat/compose.yml:rw - /var/run/docker.sock:/var/run/docker.sock + extra_hosts: + - "host.docker.internal:host-gateway" networks: - updater diff --git a/updater/git_remote.py b/updater/git_remote.py index 26cd0c9..ec09c4d 100644 --- a/updater/git_remote.py +++ b/updater/git_remote.py @@ -7,11 +7,17 @@ Gitea: git.fromchat.ru/FromChat/{same repo name} Official GitHub URLs fall back to Gitea when GitHub fails. URLs already on git.fromchat.ru use Gitea APIs directly. + +Gitea on the same host is preferred via Docker host-gateway :3000 (plain HTTP), +because public https://git.fromchat.ru goes through Caddy TLS and can fail from +containers when certs are unavailable or hairpin routing breaks. """ from __future__ import annotations import json +import os import re +import ssl import sys import urllib.error import urllib.parse @@ -22,7 +28,10 @@ SEMVER = re.compile(r"^v(\d+)\.(\d+)(?:\.(\d+))?$") GITHUB_ORG = "fromchat-messenger" GITEA_ORG = "FromChat" GITHUB_OFFICIAL_PREFIX = f"https://github.com/{GITHUB_ORG}/" -GITEA_BASE = "https://git.fromchat.ru" +GITEA_PUBLIC_HOST = "git.fromchat.ru" +GITEA_PUBLIC_BASE = f"https://{GITEA_PUBLIC_HOST}" +# Kept for callers / docs; resolved dynamically via gitea_base_candidates(). +GITEA_BASE = GITEA_PUBLIC_BASE OFFICIAL_GITHUB_REPOS = frozenset( { @@ -32,6 +41,8 @@ OFFICIAL_GITHUB_REPOS = frozenset( } ) +_gitea_working_base: str | None = None + def normalize_repo_url(url: str) -> str: u = url.rstrip("/") @@ -73,50 +84,137 @@ def parse_slug(url: str) -> tuple[str, str]: raise ValueError(f"Cannot parse repository slug from {url!r}") -def _auth_headers(token: str | None, *, gitea: bool = False) -> dict[str, str]: - headers = {"User-Agent": "fromchat-git-remote"} +def _normalize_token(token: str | None) -> str | None: if not token: - return headers + return None + cleaned = token.strip() + return cleaned or None + + +def gitea_base_candidates() -> list[str]: + """Prefer same-host Gitea (:3000) before public HTTPS (Caddy TLS).""" + out: list[str] = [] + env = os.environ.get("GITEA_BASE", "").strip().rstrip("/") + if env: + out.append(env) + # Caddyfile reverse_proxies git.fromchat.ru to host :3000 + for host in ( + "host.docker.internal:3000", + "172.17.0.1:3000", + "172.18.0.1:3000", + "172.19.0.1:3000", + "172.20.0.1:3000", + "172.21.0.1:3000", + ): + out.append(f"http://{host}") + out.append(GITEA_PUBLIC_BASE) + seen: set[str] = set() + unique: list[str] = [] + for base in out: + if base and base not in seen: + seen.add(base) + unique.append(base) + return unique + + +def _auth_headers(token: str | None, *, gitea: bool = False) -> dict[str, str]: + token = _normalize_token(token) + headers: dict[str, str] = {"User-Agent": "fromchat-git-remote"} if gitea: - # Gitea accepts both schemes; token is the documented form. - headers["Authorization"] = f"token {token}" - else: - headers["Authorization"] = f"Bearer {token}" + headers["Accept"] = "application/json" + if token: + headers["Authorization"] = f"token {token}" + return headers + headers["Accept"] = "application/vnd.github+json" + headers["X-GitHub-Api-Version"] = "2022-11-28" + if token: + if token.startswith("github_pat_") or token.startswith("gho_"): + headers["Authorization"] = f"Bearer {token}" + else: + headers["Authorization"] = f"token {token}" return headers -def _http_get(url: str, token: str | None, *, gitea: bool = False) -> bytes: - req = urllib.request.Request( - url, - headers={ - **_auth_headers(token, gitea=gitea), - **({"Accept": "application/vnd.github+json"} if not gitea else {}), - }, - ) - with urllib.request.urlopen(req, timeout=45) as resp: +def _http_get_once( + url: str, + token: str | None, + *, + gitea: bool = False, + insecure: bool = False, +) -> bytes: + headers = _auth_headers(token, gitea=gitea) + if gitea: + host = urllib.parse.urlparse(url).hostname or "" + if host and host not in (GITEA_PUBLIC_HOST, "localhost", "127.0.0.1"): + headers["Host"] = GITEA_PUBLIC_HOST + req = urllib.request.Request(url, headers=headers) + context = ssl._create_unverified_context() if insecure else None + with urllib.request.urlopen(req, timeout=45, context=context) as resp: return resp.read() +def _http_get(url: str, token: str | None, *, gitea: bool = False) -> bytes: + if not gitea: + return _http_get_once(url, token, gitea=False) + try: + return _http_get_once(url, token, gitea=True) + except Exception: + if url.startswith("https://"): + try: + return _http_get_once(url, token, gitea=True, insecure=True) + except Exception: + pass + parsed = urllib.parse.urlparse(url) + return _gitea_get(parsed.path + (f"?{parsed.query}" if parsed.query else ""), token) + + +def _gitea_get(path: str, token: str | None) -> bytes: + """GET a Gitea path, probing host-gateway HTTP then public HTTPS.""" + global _gitea_working_base + if not path.startswith("/"): + path = "/" + path + + bases: list[str] = [] + if _gitea_working_base: + bases.append(_gitea_working_base) + for candidate in gitea_base_candidates(): + if candidate not in bases: + bases.append(candidate) + + last_error: Exception | None = None + for base in bases: + url = f"{base.rstrip('/')}{path}" + insecure_opts = (False, True) if base.startswith("https://") else (False,) + for insecure in insecure_opts: + try: + data = _http_get_once(url, token, gitea=True, insecure=insecure) + if _gitea_working_base != base: + _gitea_working_base = base + if base != GITEA_PUBLIC_BASE: + print(f"Using Gitea at {base}", file=sys.stderr) + return data + except (urllib.error.HTTPError, urllib.error.URLError, TimeoutError, ssl.SSLError) as exc: + last_error = exc + continue + raise RuntimeError(f"Gitea request failed for {path}") from last_error + + def github_raw_url(owner: str, repo: str, ref: str, path: str) -> str: return f"https://raw.githubusercontent.com/{owner}/{repo}/{ref}/{path.lstrip('/')}" -def gitea_api_raw_url(gitea_owner: str, gitea_repo: str, ref: str, path: str) -> str: - """Gitea API raw file (supports branch/tag/commit via ?ref=).""" +def gitea_api_raw_path(gitea_owner: str, gitea_repo: str, ref: str, path: str) -> str: filepath = urllib.parse.quote(path.lstrip("/"), safe="/") ref_q = urllib.parse.quote(ref, safe="") - return ( - f"{GITEA_BASE}/api/v1/repos/{gitea_owner}/{gitea_repo}/raw/{filepath}?ref={ref_q}" - ) + return f"/api/v1/repos/{gitea_owner}/{gitea_repo}/raw/{filepath}?ref={ref_q}" -def gitea_web_raw_urls(gitea_owner: str, gitea_repo: str, ref: str, path: str) -> list[str]: - """Legacy/browser raw paths (tried if API raw fails).""" +def gitea_web_raw_paths(gitea_owner: str, gitea_repo: str, ref: str, path: str) -> list[str]: p = path.lstrip("/") ref_q = urllib.parse.quote(ref, safe="") return [ - f"{GITEA_BASE}/{gitea_owner}/{gitea_repo}/raw/{ref_q}/{p}", - f"{GITEA_BASE}/{gitea_owner}/{gitea_repo}/raw/tag/{ref_q}/{p}", + f"/{gitea_owner}/{gitea_repo}/raw/{ref_q}/{p}", + f"/{gitea_owner}/{gitea_repo}/raw/tag/{ref_q}/{p}", ] @@ -127,14 +225,14 @@ def fetch_gitea_raw( path: str, token: str | None, ) -> bytes: - urls = [gitea_api_raw_url(gitea_owner, gitea_repo, ref, path)] - urls.extend(gitea_web_raw_urls(gitea_owner, gitea_repo, ref, path)) + paths = [gitea_api_raw_path(gitea_owner, gitea_repo, ref, path)] + paths.extend(gitea_web_raw_paths(gitea_owner, gitea_repo, ref, path)) last_error: Exception | None = None - for url in urls: + for rel in paths: try: - return _http_get(url, token, gitea=True) - except (urllib.error.HTTPError, urllib.error.URLError, TimeoutError) as exc: + return _gitea_get(rel, token) + except (RuntimeError, urllib.error.HTTPError, urllib.error.URLError, TimeoutError) as exc: last_error = exc raise RuntimeError( f"Failed to fetch {path} from git.fromchat.ru/{gitea_owner}/{gitea_repo}@{ref}" @@ -142,6 +240,7 @@ def fetch_gitea_raw( def fetch_raw_file(repo_url: str, ref: str, path: str, token: str | None = None) -> bytes: + token = _normalize_token(token) owner, repo = parse_slug(repo_url) if is_gitea_repo(repo_url): @@ -150,7 +249,7 @@ def fetch_raw_file(repo_url: str, ref: str, path: str, token: str | None = None) try: return _http_get(github_raw_url(owner, repo, ref, path), token, gitea=False) - except (urllib.error.HTTPError, urllib.error.URLError, TimeoutError) as github_err: + except Exception as github_err: if not is_official_github_repo(repo_url): raise RuntimeError( f"Failed to fetch {path} from {repo_url}@{ref}" @@ -180,8 +279,7 @@ def _gitea_paginated_json(path: str, token: str | None) -> list[dict]: page = 1 while True: sep = "&" if "?" in path else "?" - url = f"{GITEA_BASE}{path}{sep}page={page}&limit=50" - chunk = json.loads(_http_get(url, token, gitea=True).decode()) + chunk = json.loads(_gitea_get(f"{path}{sep}page={page}&limit=50", token).decode()) if not isinstance(chunk, list): break if not chunk: @@ -207,7 +305,6 @@ def _tags_from_gitea(gitea_owner: str, gitea_repo: str, token: str | None) -> se if out: return out - # Fallback: release tags for item in _gitea_paginated_json( f"/api/v1/repos/{gitea_owner}/{gitea_repo}/releases", token, @@ -220,6 +317,7 @@ def _tags_from_gitea(gitea_owner: str, gitea_repo: str, token: str | None) -> se def fetch_semver_tags(repo_url: str, token: str | None = None) -> set[str]: + token = _normalize_token(token) owner, repo = parse_slug(repo_url) if is_gitea_repo(repo_url): @@ -228,7 +326,7 @@ def fetch_semver_tags(repo_url: str, token: str | None = None) -> set[str]: try: return _tags_from_github(owner, repo, token) - except (urllib.error.HTTPError, urllib.error.URLError, TimeoutError, json.JSONDecodeError): + except Exception: if not is_official_github_repo(repo_url): raise gitea_owner, gitea_repo = gitea_owner_repo(owner, repo) @@ -293,7 +391,7 @@ def package_exists_gitea(gitea_owner: str, package: str, tag: str, token: str) - path = f"/api/v1/packages/{gitea_owner}/container/{short}/versions" try: versions = _gitea_paginated_json(path, token) - except (urllib.error.HTTPError, urllib.error.URLError, json.JSONDecodeError): + except (RuntimeError, urllib.error.HTTPError, urllib.error.URLError, json.JSONDecodeError): return False for version in versions: @@ -342,9 +440,11 @@ def package_version_exists( def resolve_git_token() -> str | None: - import os - - return os.environ.get("GIT_TOKEN") or os.environ.get("GITHUB_TOKEN") or None + return _normalize_token( + os.environ.get("UPDATER_TOKEN") + or os.environ.get("GIT_TOKEN") + or os.environ.get("GITHUB_TOKEN") + ) def main() -> None: diff --git a/updater/github_client.py b/updater/github_client.py index f18d4cf..726835c 100644 --- a/updater/github_client.py +++ b/updater/github_client.py @@ -54,7 +54,12 @@ class Settings: for c in os.environ.get("FROMCHAT_COMPONENTS", "backend,frontend").split(",") if c.strip() ] - token = os.environ.get("GIT_TOKEN") or os.environ.get("GITHUB_TOKEN") or "" + token = ( + os.environ.get("UPDATER_TOKEN") + or os.environ.get("GIT_TOKEN") + or os.environ.get("GITHUB_TOKEN") + or "" + ).strip() backend = os.environ["BACKEND_REPO"] owner, _ = parse_slug(backend) current = read_version_from_compose(COMPOSE_FILE) diff --git a/updater/main.py b/updater/main.py index 1f97c37..522c5d9 100644 --- a/updater/main.py +++ b/updater/main.py @@ -14,6 +14,11 @@ logger = logging.getLogger("fromchat.updater.main") def main() -> None: settings = Settings.from_env() + if not settings.github_token: + logger.warning( + "UPDATER_TOKEN is not set (check server .env and updater env_file); " + "GitHub/Gitea API calls will fail" + ) logger.info( "FromChat updater started (project=%s, current=%s, interval=%ss)", settings.compose_project_dir,