This commit is contained in:
2026-07-14 17:53:07 +03:00
Unverified
parent d100a21020
commit 26555a6a4c
6 changed files with 172 additions and 57 deletions
+3 -2
View File
@@ -1,11 +1,12 @@
GITHUB_TOKEN= # Repo/config for the updater. UPDATER_TOKEN lives in deployment/.env.prod (copied to server .env).
GIT_TOKEN=
BACKEND_REPO=https://github.com/fromchat-messenger/backend.git BACKEND_REPO=https://github.com/fromchat-messenger/backend.git
WEB_REPO=https://github.com/fromchat-messenger/web.git WEB_REPO=https://github.com/fromchat-messenger/web.git
DEPLOYMENT_REPO=https://github.com/fromchat-messenger/app.git DEPLOYMENT_REPO=https://github.com/fromchat-messenger/app.git
COMPOSE_PROJECT_DIR=/home/user/fromchat-server COMPOSE_PROJECT_DIR=/home/user/fromchat-server
FROMCHAT_COMPONENTS=backend,frontend,caddy FROMCHAT_COMPONENTS=backend,frontend,caddy
CHECK_INTERVAL_SECONDS=60 CHECK_INTERVAL_SECONDS=60
# Optional: force Gitea API base (otherwise host-gateway :3000 is probed first)
# GITEA_BASE=http://host.docker.internal:3000
# Gitea-only example (no GitHub fallback needed): # Gitea-only example (no GitHub fallback needed):
# BACKEND_REPO=https://git.fromchat.ru/FromChat/backend.git # BACKEND_REPO=https://git.fromchat.ru/FromChat/backend.git
+13 -13
View File
@@ -4,22 +4,22 @@ Watches GitHub **tags** and **container packages** for backend + frontend repos.
When a newer semver tag exists in both and all required `fromchat/*` images are published, When a newer semver tag exists in both and all required `fromchat/*` images are published,
regenerates `compose.yml` (via the deployment merge script) and restarts the stack. regenerates `compose.yml` (via the deployment merge script) and restarts the stack.
## Environment (`.env`) ## Environment
| Variable | Description | Production stack: the updater service loads **both** the main stack `.env` (secrets from `deployment/.env.prod`) and `updater/.env` (repo URLs and paths below).
|----------|-------------|
| `GITHUB_TOKEN` | PAT with `read:packages` + `repo` |
| `BACKEND_REPO` | e.g. `https://github.com/fromchat-messenger/backend.git` |
| `WEB_REPO` | e.g. `https://github.com/fromchat-messenger/web.git` |
| `DEPLOYMENT_REPO` | `https://github.com/fromchat-messenger/app.git` (deployment scripts) |
| `DEPLOYMENT_REPO` | Deployment repo for `generate-compose.py` |
| `COMPOSE_PROJECT_DIR` | Host path to the stack (`compose.yml` parent directory) |
| `FROMCHAT_COMPONENTS` | Comma list: `backend,frontend,caddy,updater` |
| `CHECK_INTERVAL_SECONDS` | Poll interval (default `60`) |
Only `compose.yml` from the stack directory is mounted into the updater container (read-write at `/fromchat/compose.yml`). The current release is read from image tags inside that file. | Variable | Where | Description |
|----------|-------|-------------|
| `UPDATER_TOKEN` | main `.env` | PAT with `read:packages` + `repo` (GitHub or Gitea) |
| `BACKEND_REPO` | `updater/.env` | e.g. `https://github.com/fromchat-messenger/backend.git` |
| `WEB_REPO` | `updater/.env` | e.g. `https://github.com/fromchat-messenger/web.git` |
| `DEPLOYMENT_REPO` | `updater/.env` | Deployment repo for `generate-compose.py` |
| `COMPOSE_PROJECT_DIR` | `updater/.env` | Host path to the stack (`compose.yml` parent directory) |
| `FROMCHAT_COMPONENTS` | `updater/.env` | Comma list: `backend,frontend,caddy,updater` |
| `CHECK_INTERVAL_SECONDS` | `updater/.env` | Poll interval (default `60`) |
| `GITEA_BASE` | optional | Override Gitea API base (e.g. `http://host.docker.internal:3000`). If unset, probes host-gateway `:3000` then `https://git.fromchat.ru`. |
Create token: [GitHub PAT (read:packages + repo)](https://github.com/settings/tokens/new?description=FromChat%20Updater&scopes=read:packages,repo) Create token: [GitHub PAT (read:packages + repo)](https://github.com/settings/tokens/new?description=FromChat%20Updater&scopes=read:packages,repo) — set as `UPDATER_TOKEN` in `deployment/.env.prod` (deploy copies it to the server main `.env`).
## Run standalone ## Run standalone
+5 -1
View File
@@ -1,12 +1,16 @@
services: services:
updater: updater:
image: fromchat/updater:latest build:
context: .
dockerfile: Dockerfile
restart: unless-stopped restart: unless-stopped
env_file: env_file:
- .env - .env
volumes: volumes:
- ${COMPOSE_PROJECT_DIR}/compose.yml:/fromchat/compose.yml:rw - ${COMPOSE_PROJECT_DIR}/compose.yml:/fromchat/compose.yml:rw
- /var/run/docker.sock:/var/run/docker.sock - /var/run/docker.sock:/var/run/docker.sock
extra_hosts:
- "host.docker.internal:host-gateway"
networks: networks:
- updater - updater
+139 -39
View File
@@ -7,11 +7,17 @@ Gitea: git.fromchat.ru/FromChat/{same repo name}
Official GitHub URLs fall back to Gitea when GitHub fails. Official GitHub URLs fall back to Gitea when GitHub fails.
URLs already on git.fromchat.ru use Gitea APIs directly. URLs already on git.fromchat.ru use Gitea APIs directly.
Gitea on the same host is preferred via Docker host-gateway :3000 (plain HTTP),
because public https://git.fromchat.ru goes through Caddy TLS and can fail from
containers when certs are unavailable or hairpin routing breaks.
""" """
from __future__ import annotations from __future__ import annotations
import json import json
import os
import re import re
import ssl
import sys import sys
import urllib.error import urllib.error
import urllib.parse import urllib.parse
@@ -22,7 +28,10 @@ SEMVER = re.compile(r"^v(\d+)\.(\d+)(?:\.(\d+))?$")
GITHUB_ORG = "fromchat-messenger" GITHUB_ORG = "fromchat-messenger"
GITEA_ORG = "FromChat" GITEA_ORG = "FromChat"
GITHUB_OFFICIAL_PREFIX = f"https://github.com/{GITHUB_ORG}/" GITHUB_OFFICIAL_PREFIX = f"https://github.com/{GITHUB_ORG}/"
GITEA_BASE = "https://git.fromchat.ru" GITEA_PUBLIC_HOST = "git.fromchat.ru"
GITEA_PUBLIC_BASE = f"https://{GITEA_PUBLIC_HOST}"
# Kept for callers / docs; resolved dynamically via gitea_base_candidates().
GITEA_BASE = GITEA_PUBLIC_BASE
OFFICIAL_GITHUB_REPOS = frozenset( OFFICIAL_GITHUB_REPOS = frozenset(
{ {
@@ -32,6 +41,8 @@ OFFICIAL_GITHUB_REPOS = frozenset(
} }
) )
_gitea_working_base: str | None = None
def normalize_repo_url(url: str) -> str: def normalize_repo_url(url: str) -> str:
u = url.rstrip("/") u = url.rstrip("/")
@@ -73,50 +84,137 @@ def parse_slug(url: str) -> tuple[str, str]:
raise ValueError(f"Cannot parse repository slug from {url!r}") raise ValueError(f"Cannot parse repository slug from {url!r}")
def _auth_headers(token: str | None, *, gitea: bool = False) -> dict[str, str]: def _normalize_token(token: str | None) -> str | None:
headers = {"User-Agent": "fromchat-git-remote"}
if not token: if not token:
return headers return None
cleaned = token.strip()
return cleaned or None
def gitea_base_candidates() -> list[str]:
"""Prefer same-host Gitea (:3000) before public HTTPS (Caddy TLS)."""
out: list[str] = []
env = os.environ.get("GITEA_BASE", "").strip().rstrip("/")
if env:
out.append(env)
# Caddyfile reverse_proxies git.fromchat.ru to host :3000
for host in (
"host.docker.internal:3000",
"172.17.0.1:3000",
"172.18.0.1:3000",
"172.19.0.1:3000",
"172.20.0.1:3000",
"172.21.0.1:3000",
):
out.append(f"http://{host}")
out.append(GITEA_PUBLIC_BASE)
seen: set[str] = set()
unique: list[str] = []
for base in out:
if base and base not in seen:
seen.add(base)
unique.append(base)
return unique
def _auth_headers(token: str | None, *, gitea: bool = False) -> dict[str, str]:
token = _normalize_token(token)
headers: dict[str, str] = {"User-Agent": "fromchat-git-remote"}
if gitea: if gitea:
# Gitea accepts both schemes; token is the documented form. headers["Accept"] = "application/json"
if token:
headers["Authorization"] = f"token {token}" headers["Authorization"] = f"token {token}"
else:
headers["Authorization"] = f"Bearer {token}"
return headers return headers
headers["Accept"] = "application/vnd.github+json"
headers["X-GitHub-Api-Version"] = "2022-11-28"
if token:
if token.startswith("github_pat_") or token.startswith("gho_"):
headers["Authorization"] = f"Bearer {token}"
else:
headers["Authorization"] = f"token {token}"
return headers
def _http_get_once(
url: str,
token: str | None,
*,
gitea: bool = False,
insecure: bool = False,
) -> bytes:
headers = _auth_headers(token, gitea=gitea)
if gitea:
host = urllib.parse.urlparse(url).hostname or ""
if host and host not in (GITEA_PUBLIC_HOST, "localhost", "127.0.0.1"):
headers["Host"] = GITEA_PUBLIC_HOST
req = urllib.request.Request(url, headers=headers)
context = ssl._create_unverified_context() if insecure else None
with urllib.request.urlopen(req, timeout=45, context=context) as resp:
return resp.read()
def _http_get(url: str, token: str | None, *, gitea: bool = False) -> bytes: def _http_get(url: str, token: str | None, *, gitea: bool = False) -> bytes:
req = urllib.request.Request( if not gitea:
url, return _http_get_once(url, token, gitea=False)
headers={ try:
**_auth_headers(token, gitea=gitea), return _http_get_once(url, token, gitea=True)
**({"Accept": "application/vnd.github+json"} if not gitea else {}), except Exception:
}, if url.startswith("https://"):
) try:
with urllib.request.urlopen(req, timeout=45) as resp: return _http_get_once(url, token, gitea=True, insecure=True)
return resp.read() except Exception:
pass
parsed = urllib.parse.urlparse(url)
return _gitea_get(parsed.path + (f"?{parsed.query}" if parsed.query else ""), token)
def _gitea_get(path: str, token: str | None) -> bytes:
"""GET a Gitea path, probing host-gateway HTTP then public HTTPS."""
global _gitea_working_base
if not path.startswith("/"):
path = "/" + path
bases: list[str] = []
if _gitea_working_base:
bases.append(_gitea_working_base)
for candidate in gitea_base_candidates():
if candidate not in bases:
bases.append(candidate)
last_error: Exception | None = None
for base in bases:
url = f"{base.rstrip('/')}{path}"
insecure_opts = (False, True) if base.startswith("https://") else (False,)
for insecure in insecure_opts:
try:
data = _http_get_once(url, token, gitea=True, insecure=insecure)
if _gitea_working_base != base:
_gitea_working_base = base
if base != GITEA_PUBLIC_BASE:
print(f"Using Gitea at {base}", file=sys.stderr)
return data
except (urllib.error.HTTPError, urllib.error.URLError, TimeoutError, ssl.SSLError) as exc:
last_error = exc
continue
raise RuntimeError(f"Gitea request failed for {path}") from last_error
def github_raw_url(owner: str, repo: str, ref: str, path: str) -> str: def github_raw_url(owner: str, repo: str, ref: str, path: str) -> str:
return f"https://raw.githubusercontent.com/{owner}/{repo}/{ref}/{path.lstrip('/')}" return f"https://raw.githubusercontent.com/{owner}/{repo}/{ref}/{path.lstrip('/')}"
def gitea_api_raw_url(gitea_owner: str, gitea_repo: str, ref: str, path: str) -> str: def gitea_api_raw_path(gitea_owner: str, gitea_repo: str, ref: str, path: str) -> str:
"""Gitea API raw file (supports branch/tag/commit via ?ref=)."""
filepath = urllib.parse.quote(path.lstrip("/"), safe="/") filepath = urllib.parse.quote(path.lstrip("/"), safe="/")
ref_q = urllib.parse.quote(ref, safe="") ref_q = urllib.parse.quote(ref, safe="")
return ( return f"/api/v1/repos/{gitea_owner}/{gitea_repo}/raw/{filepath}?ref={ref_q}"
f"{GITEA_BASE}/api/v1/repos/{gitea_owner}/{gitea_repo}/raw/{filepath}?ref={ref_q}"
)
def gitea_web_raw_urls(gitea_owner: str, gitea_repo: str, ref: str, path: str) -> list[str]: def gitea_web_raw_paths(gitea_owner: str, gitea_repo: str, ref: str, path: str) -> list[str]:
"""Legacy/browser raw paths (tried if API raw fails)."""
p = path.lstrip("/") p = path.lstrip("/")
ref_q = urllib.parse.quote(ref, safe="") ref_q = urllib.parse.quote(ref, safe="")
return [ return [
f"{GITEA_BASE}/{gitea_owner}/{gitea_repo}/raw/{ref_q}/{p}", f"/{gitea_owner}/{gitea_repo}/raw/{ref_q}/{p}",
f"{GITEA_BASE}/{gitea_owner}/{gitea_repo}/raw/tag/{ref_q}/{p}", f"/{gitea_owner}/{gitea_repo}/raw/tag/{ref_q}/{p}",
] ]
@@ -127,14 +225,14 @@ def fetch_gitea_raw(
path: str, path: str,
token: str | None, token: str | None,
) -> bytes: ) -> bytes:
urls = [gitea_api_raw_url(gitea_owner, gitea_repo, ref, path)] paths = [gitea_api_raw_path(gitea_owner, gitea_repo, ref, path)]
urls.extend(gitea_web_raw_urls(gitea_owner, gitea_repo, ref, path)) paths.extend(gitea_web_raw_paths(gitea_owner, gitea_repo, ref, path))
last_error: Exception | None = None last_error: Exception | None = None
for url in urls: for rel in paths:
try: try:
return _http_get(url, token, gitea=True) return _gitea_get(rel, token)
except (urllib.error.HTTPError, urllib.error.URLError, TimeoutError) as exc: except (RuntimeError, urllib.error.HTTPError, urllib.error.URLError, TimeoutError) as exc:
last_error = exc last_error = exc
raise RuntimeError( raise RuntimeError(
f"Failed to fetch {path} from git.fromchat.ru/{gitea_owner}/{gitea_repo}@{ref}" f"Failed to fetch {path} from git.fromchat.ru/{gitea_owner}/{gitea_repo}@{ref}"
@@ -142,6 +240,7 @@ def fetch_gitea_raw(
def fetch_raw_file(repo_url: str, ref: str, path: str, token: str | None = None) -> bytes: def fetch_raw_file(repo_url: str, ref: str, path: str, token: str | None = None) -> bytes:
token = _normalize_token(token)
owner, repo = parse_slug(repo_url) owner, repo = parse_slug(repo_url)
if is_gitea_repo(repo_url): if is_gitea_repo(repo_url):
@@ -150,7 +249,7 @@ def fetch_raw_file(repo_url: str, ref: str, path: str, token: str | None = None)
try: try:
return _http_get(github_raw_url(owner, repo, ref, path), token, gitea=False) return _http_get(github_raw_url(owner, repo, ref, path), token, gitea=False)
except (urllib.error.HTTPError, urllib.error.URLError, TimeoutError) as github_err: except Exception as github_err:
if not is_official_github_repo(repo_url): if not is_official_github_repo(repo_url):
raise RuntimeError( raise RuntimeError(
f"Failed to fetch {path} from {repo_url}@{ref}" f"Failed to fetch {path} from {repo_url}@{ref}"
@@ -180,8 +279,7 @@ def _gitea_paginated_json(path: str, token: str | None) -> list[dict]:
page = 1 page = 1
while True: while True:
sep = "&" if "?" in path else "?" sep = "&" if "?" in path else "?"
url = f"{GITEA_BASE}{path}{sep}page={page}&limit=50" chunk = json.loads(_gitea_get(f"{path}{sep}page={page}&limit=50", token).decode())
chunk = json.loads(_http_get(url, token, gitea=True).decode())
if not isinstance(chunk, list): if not isinstance(chunk, list):
break break
if not chunk: if not chunk:
@@ -207,7 +305,6 @@ def _tags_from_gitea(gitea_owner: str, gitea_repo: str, token: str | None) -> se
if out: if out:
return out return out
# Fallback: release tags
for item in _gitea_paginated_json( for item in _gitea_paginated_json(
f"/api/v1/repos/{gitea_owner}/{gitea_repo}/releases", f"/api/v1/repos/{gitea_owner}/{gitea_repo}/releases",
token, token,
@@ -220,6 +317,7 @@ def _tags_from_gitea(gitea_owner: str, gitea_repo: str, token: str | None) -> se
def fetch_semver_tags(repo_url: str, token: str | None = None) -> set[str]: def fetch_semver_tags(repo_url: str, token: str | None = None) -> set[str]:
token = _normalize_token(token)
owner, repo = parse_slug(repo_url) owner, repo = parse_slug(repo_url)
if is_gitea_repo(repo_url): if is_gitea_repo(repo_url):
@@ -228,7 +326,7 @@ def fetch_semver_tags(repo_url: str, token: str | None = None) -> set[str]:
try: try:
return _tags_from_github(owner, repo, token) return _tags_from_github(owner, repo, token)
except (urllib.error.HTTPError, urllib.error.URLError, TimeoutError, json.JSONDecodeError): except Exception:
if not is_official_github_repo(repo_url): if not is_official_github_repo(repo_url):
raise raise
gitea_owner, gitea_repo = gitea_owner_repo(owner, repo) gitea_owner, gitea_repo = gitea_owner_repo(owner, repo)
@@ -293,7 +391,7 @@ def package_exists_gitea(gitea_owner: str, package: str, tag: str, token: str) -
path = f"/api/v1/packages/{gitea_owner}/container/{short}/versions" path = f"/api/v1/packages/{gitea_owner}/container/{short}/versions"
try: try:
versions = _gitea_paginated_json(path, token) versions = _gitea_paginated_json(path, token)
except (urllib.error.HTTPError, urllib.error.URLError, json.JSONDecodeError): except (RuntimeError, urllib.error.HTTPError, urllib.error.URLError, json.JSONDecodeError):
return False return False
for version in versions: for version in versions:
@@ -342,9 +440,11 @@ def package_version_exists(
def resolve_git_token() -> str | None: def resolve_git_token() -> str | None:
import os return _normalize_token(
os.environ.get("UPDATER_TOKEN")
return os.environ.get("GIT_TOKEN") or os.environ.get("GITHUB_TOKEN") or None or os.environ.get("GIT_TOKEN")
or os.environ.get("GITHUB_TOKEN")
)
def main() -> None: def main() -> None:
+6 -1
View File
@@ -54,7 +54,12 @@ class Settings:
for c in os.environ.get("FROMCHAT_COMPONENTS", "backend,frontend").split(",") for c in os.environ.get("FROMCHAT_COMPONENTS", "backend,frontend").split(",")
if c.strip() if c.strip()
] ]
token = os.environ.get("GIT_TOKEN") or os.environ.get("GITHUB_TOKEN") or "" token = (
os.environ.get("UPDATER_TOKEN")
or os.environ.get("GIT_TOKEN")
or os.environ.get("GITHUB_TOKEN")
or ""
).strip()
backend = os.environ["BACKEND_REPO"] backend = os.environ["BACKEND_REPO"]
owner, _ = parse_slug(backend) owner, _ = parse_slug(backend)
current = read_version_from_compose(COMPOSE_FILE) current = read_version_from_compose(COMPOSE_FILE)
+5
View File
@@ -14,6 +14,11 @@ logger = logging.getLogger("fromchat.updater.main")
def main() -> None: def main() -> None:
settings = Settings.from_env() settings = Settings.from_env()
if not settings.github_token:
logger.warning(
"UPDATER_TOKEN is not set (check server .env and updater env_file); "
"GitHub/Gitea API calls will fail"
)
logger.info( logger.info(
"FromChat updater started (project=%s, current=%s, interval=%ss)", "FromChat updater started (project=%s, current=%s, interval=%ss)",
settings.compose_project_dir, settings.compose_project_dir,