mirror of
https://github.com/fromchat-messenger/deployment.git
synced 2026-11-20 10:16:32 +03:00
Improve auto-gen workflow
This commit is contained in:
@@ -291,7 +291,11 @@ def _generate_compose(
|
||||
if keep_build:
|
||||
cmd.append("--keep-build")
|
||||
if "backend" in components and paths.backend_dir:
|
||||
cmd.extend(["--backend-compose", str(paths.backend_dir / "compose.yml")])
|
||||
backend_compose = paths.backend_dir / "compose.yml"
|
||||
cmd.extend(["--backend-compose", str(backend_compose)])
|
||||
backend_prod = paths.backend_dir / "compose.prod.yml"
|
||||
if backend_prod.is_file():
|
||||
cmd.extend(["--backend-compose-prod", str(backend_prod)])
|
||||
cmd.extend(["--backend-root", str(paths.backend_dir)])
|
||||
if "frontend" in components and paths.web_dir:
|
||||
cmd.extend(["--frontend-compose", str(paths.web_dir / "compose.yml")])
|
||||
|
||||
@@ -245,6 +245,9 @@ def discover_image_specs(paths: ProjectPaths) -> list[ImageSpec]:
|
||||
compose_files: list[tuple[Path, Path]] = []
|
||||
if paths.backend_dir and (paths.backend_dir / "compose.yml").is_file():
|
||||
compose_files.append((paths.backend_dir, paths.backend_dir / "compose.yml"))
|
||||
prod_compose = paths.backend_dir / "compose.prod.yml"
|
||||
if prod_compose.is_file():
|
||||
compose_files.append((paths.backend_dir, prod_compose))
|
||||
if paths.web_dir and (paths.web_dir / "compose.yml").is_file():
|
||||
compose_files.append((paths.web_dir, paths.web_dir / "compose.yml"))
|
||||
if paths.updater_dir and (paths.updater_dir / "compose.yml").is_file():
|
||||
|
||||
+30
-56
@@ -1,15 +1,13 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Merge backend (+ frontend / updater) compose for production deploy.
|
||||
Uses backend compose as the source of truth (including caddy + livekit).
|
||||
Uses backend compose.yml as the base; merges compose.prod.yml when caddy is selected.
|
||||
Replaces build: with image: fromchat/<service>:<tag>.
|
||||
When caddy is not selected, the caddy service is left commented out in the file.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import copy
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
@@ -32,7 +30,7 @@ FROMCHAT_IMAGE_SERVICES = frozenset(
|
||||
)
|
||||
|
||||
BACKEND_SERVICES = frozenset(
|
||||
{"backend", "messaging", "file_storage", "livekit", "postgres", "caddy", "haproxy"}
|
||||
{"backend", "messaging", "file_storage", "livekit", "postgres"}
|
||||
)
|
||||
FRONTEND_SERVICES = frozenset({"web"})
|
||||
UPDATER_SERVICES = frozenset({"updater"})
|
||||
@@ -141,6 +139,19 @@ def _rewrite_build_context(build: dict[str, Any], root: Path) -> None:
|
||||
build["dockerfile"] = dockerfile
|
||||
|
||||
|
||||
def resolve_backend_compose_prod(
|
||||
backend_compose: Path | None,
|
||||
explicit: Path | None,
|
||||
) -> Path | None:
|
||||
if explicit is not None:
|
||||
return explicit if explicit.is_file() else None
|
||||
if backend_compose is not None and backend_compose.is_file():
|
||||
candidate = backend_compose.parent / "compose.prod.yml"
|
||||
if candidate.is_file():
|
||||
return candidate
|
||||
return None
|
||||
|
||||
|
||||
def load_updater_service(updater_root: Path) -> dict[str, Any]:
|
||||
compose_path = updater_root / "compose.yml"
|
||||
if compose_path.is_file():
|
||||
@@ -187,37 +198,10 @@ def filter_services(
|
||||
return {name: cfg for name, cfg in services.items() if name in enabled}
|
||||
|
||||
|
||||
def comment_out_service(yaml_text: str, service_name: str) -> str:
|
||||
"""Comment out a top-level service block (two-space indent under services:)."""
|
||||
lines = yaml_text.splitlines(keepends=True)
|
||||
out: list[str] = []
|
||||
commenting = False
|
||||
header = re.compile(rf"^ {re.escape(service_name)}:\s*(#.*)?$")
|
||||
next_svc = re.compile(r"^ [A-Za-z0-9_-]+:\s*(#.*)?$")
|
||||
top_key = re.compile(r"^[A-Za-z0-9_-]+:\s*(#.*)?$")
|
||||
|
||||
for line in lines:
|
||||
if not commenting and header.match(line.rstrip("\n")):
|
||||
commenting = True
|
||||
out.append(f"# {line}" if not line.startswith("#") else line)
|
||||
continue
|
||||
if commenting:
|
||||
if next_svc.match(line.rstrip("\n")) or top_key.match(line.rstrip("\n")):
|
||||
commenting = False
|
||||
out.append(line)
|
||||
continue
|
||||
if line.startswith("#") or line.strip() == "":
|
||||
out.append(line)
|
||||
else:
|
||||
out.append(f"# {line}" if not line.startswith("#") else line)
|
||||
continue
|
||||
out.append(line)
|
||||
return "".join(out)
|
||||
|
||||
|
||||
def generate(
|
||||
*,
|
||||
backend_compose: Path | None,
|
||||
backend_compose_prod: Path | None = None,
|
||||
frontend_compose: Path | None,
|
||||
tag: str,
|
||||
components: set[str],
|
||||
@@ -234,25 +218,24 @@ def generate(
|
||||
enabled: set[str] = set()
|
||||
if "backend" in components:
|
||||
enabled |= BACKEND_SERVICES
|
||||
if not include_caddy:
|
||||
enabled -= CADDY_STACK_SERVICES
|
||||
if include_caddy:
|
||||
enabled |= CADDY_STACK_SERVICES
|
||||
if "frontend" in components:
|
||||
enabled |= FRONTEND_SERVICES
|
||||
if include_updater:
|
||||
enabled |= UPDATER_SERVICES
|
||||
|
||||
# Keep caddy/haproxy in the dumped file so we can comment them out (production).
|
||||
comment_caddy = (
|
||||
not keep_build
|
||||
and "backend" in components
|
||||
and not include_caddy
|
||||
)
|
||||
if comment_caddy:
|
||||
enabled |= CADDY_STACK_SERVICES
|
||||
|
||||
sources: list[Path] = []
|
||||
if backend_compose and backend_compose.is_file():
|
||||
sources.append(backend_compose)
|
||||
if include_caddy:
|
||||
prod_compose = resolve_backend_compose_prod(backend_compose, backend_compose_prod)
|
||||
if prod_compose is None:
|
||||
raise SystemExit(
|
||||
"caddy component selected but backend compose.prod.yml was not found "
|
||||
"(pass --backend-compose-prod or place it next to compose.yml)"
|
||||
)
|
||||
sources.append(prod_compose)
|
||||
if frontend_compose and frontend_compose.is_file():
|
||||
sources.append(frontend_compose)
|
||||
|
||||
@@ -318,20 +301,9 @@ def generate(
|
||||
if not merged.get("volumes"):
|
||||
merged.pop("volumes", None)
|
||||
|
||||
text = dump_yaml_str(merged)
|
||||
if comment_caddy:
|
||||
commented = 0
|
||||
for svc in ("haproxy", "caddy"):
|
||||
if svc in services:
|
||||
text = comment_out_service(text, svc)
|
||||
commented += 1
|
||||
active = len(services) - commented
|
||||
else:
|
||||
active = len(services)
|
||||
|
||||
output.parent.mkdir(parents=True, exist_ok=True)
|
||||
output.write_text(text, encoding="utf-8")
|
||||
print(f"Wrote {output} ({active} services, tag {tag})", file=sys.stderr)
|
||||
output.write_text(dump_yaml_str(merged), encoding="utf-8")
|
||||
print(f"Wrote {output} ({len(services)} services, tag {tag})", file=sys.stderr)
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
@@ -343,6 +315,7 @@ def parse_args() -> argparse.Namespace:
|
||||
help="Comma-separated: backend,frontend,caddy",
|
||||
)
|
||||
parser.add_argument("--backend-compose", type=Path, default=None)
|
||||
parser.add_argument("--backend-compose-prod", type=Path, default=None)
|
||||
parser.add_argument("--frontend-compose", type=Path, default=None)
|
||||
parser.add_argument("--output", type=Path, default=Path("compose.yml"))
|
||||
parser.add_argument(
|
||||
@@ -362,6 +335,7 @@ def main() -> None:
|
||||
components = {c.strip().lower() for c in args.components.split(",") if c.strip()}
|
||||
generate(
|
||||
backend_compose=args.backend_compose,
|
||||
backend_compose_prod=args.backend_compose_prod,
|
||||
frontend_compose=args.frontend_compose,
|
||||
tag=args.tag,
|
||||
components=components,
|
||||
|
||||
+3
-52
@@ -12,7 +12,6 @@ from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
import ssl
|
||||
import sys
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
@@ -34,41 +33,6 @@ OFFICIAL_GITHUB_REPOS = frozenset(
|
||||
)
|
||||
|
||||
|
||||
class GitRemoteUnavailableError(OSError):
|
||||
"""GitHub and/or Gitea could not provide release metadata."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
repo_label: str,
|
||||
*,
|
||||
github_error: Exception | None = None,
|
||||
gitea_error: Exception | None = None,
|
||||
) -> None:
|
||||
self.repo_label = repo_label
|
||||
self.github_error = github_error
|
||||
self.gitea_error = gitea_error
|
||||
details: list[str] = []
|
||||
if github_error is not None:
|
||||
details.append(f"GitHub: {_brief_http_error(github_error)}")
|
||||
if gitea_error is not None:
|
||||
details.append(f"Gitea: {_brief_http_error(gitea_error)}")
|
||||
detail_text = "; ".join(details) if details else "service unavailable"
|
||||
super().__init__(f"Release metadata unavailable for {repo_label} ({detail_text})")
|
||||
|
||||
|
||||
def _brief_http_error(exc: Exception) -> str:
|
||||
if isinstance(exc, urllib.error.HTTPError):
|
||||
return f"HTTP {exc.code}"
|
||||
if isinstance(exc, urllib.error.URLError):
|
||||
reason = exc.reason
|
||||
if isinstance(reason, ssl.SSLError):
|
||||
return f"SSL {reason.reason}"
|
||||
if reason is not None:
|
||||
return str(reason)
|
||||
return "connection failed"
|
||||
return type(exc).__name__
|
||||
|
||||
|
||||
def normalize_repo_url(url: str) -> str:
|
||||
u = url.rstrip("/")
|
||||
if u.endswith(".git"):
|
||||
@@ -272,17 +236,11 @@ def fetch_semver_tags(repo_url: str, token: str | None = None) -> set[str]:
|
||||
|
||||
if is_gitea_repo(repo_url):
|
||||
gitea_owner, gitea_repo = gitea_owner_repo(owner, repo)
|
||||
try:
|
||||
return _tags_from_gitea(gitea_owner, gitea_repo, token)
|
||||
except Exception as err:
|
||||
raise GitRemoteUnavailableError(
|
||||
f"git.fromchat.ru/{gitea_owner}/{gitea_repo}",
|
||||
gitea_error=err,
|
||||
) from err
|
||||
return _tags_from_gitea(gitea_owner, gitea_repo, token)
|
||||
|
||||
try:
|
||||
return _tags_from_github(owner, repo, token)
|
||||
except Exception as github_err:
|
||||
except Exception as err:
|
||||
if not is_official_github_repo(repo_url):
|
||||
raise
|
||||
gitea_owner, gitea_repo = gitea_owner_repo(owner, repo)
|
||||
@@ -291,14 +249,7 @@ def fetch_semver_tags(repo_url: str, token: str | None = None) -> set[str]:
|
||||
f"trying git.fromchat.ru/{gitea_owner}/{gitea_repo}…",
|
||||
file=sys.stderr,
|
||||
)
|
||||
try:
|
||||
return _tags_from_gitea(gitea_owner, gitea_repo, token)
|
||||
except Exception as gitea_err:
|
||||
raise GitRemoteUnavailableError(
|
||||
f"{owner}/{repo}",
|
||||
github_error=github_err,
|
||||
gitea_error=gitea_err,
|
||||
) from gitea_err
|
||||
return _tags_from_gitea(gitea_owner, gitea_repo, token)
|
||||
|
||||
|
||||
def semver_key(tag: str) -> tuple[int, ...]:
|
||||
|
||||
+16
-2
@@ -191,7 +191,7 @@ print_success_banner() {
|
||||
|
||||
# Merge component compose files into ${install_dir}/compose.yml (image: fromchat/*:<tag>).
|
||||
# Optional local compose paths (skip remote fetch when set):
|
||||
# LOCAL_BACKEND_COMPOSE, LOCAL_FRONTEND_COMPOSE
|
||||
# LOCAL_BACKEND_COMPOSE, LOCAL_BACKEND_COMPOSE_PROD, LOCAL_FRONTEND_COMPOSE
|
||||
# Requires: DEPLOYMENT_ROOT, BACKEND_REPO, WEB_REPO (for remote fetch).
|
||||
run_generate_compose() {
|
||||
local install_dir="$1"
|
||||
@@ -202,9 +202,10 @@ run_generate_compose() {
|
||||
|
||||
step "Generating compose.yml for tag ${tag} (components: ${components_csv})…"
|
||||
|
||||
local need_backend=false need_frontend=false
|
||||
local need_backend=false need_frontend=false need_caddy=false
|
||||
[[ ",${components_csv}," == *,backend,* ]] && need_backend=true
|
||||
[[ ",${components_csv}," == *,frontend,* ]] && need_frontend=true
|
||||
[[ ",${components_csv}," == *,caddy,* ]] && need_caddy=true
|
||||
|
||||
if ${need_backend}; then
|
||||
if [[ -n "${LOCAL_BACKEND_COMPOSE:-}" && -f "${LOCAL_BACKEND_COMPOSE}" ]]; then
|
||||
@@ -215,6 +216,16 @@ run_generate_compose() {
|
||||
"${tmp}/backend.compose.yml"
|
||||
success "Backend compose.yml"
|
||||
fi
|
||||
if ${need_caddy}; then
|
||||
if [[ -n "${LOCAL_BACKEND_COMPOSE_PROD:-}" && -f "${LOCAL_BACKEND_COMPOSE_PROD}" ]]; then
|
||||
cp -f "${LOCAL_BACKEND_COMPOSE_PROD}" "${tmp}/backend.compose.prod.yml"
|
||||
success "Backend compose.prod.yml (local)"
|
||||
else
|
||||
fetch_raw_file "${BACKEND_REPO}" "${tag}" "compose.prod.yml" \
|
||||
"${tmp}/backend.compose.prod.yml"
|
||||
success "Backend compose.prod.yml"
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
if ${need_frontend}; then
|
||||
if [[ -n "${LOCAL_FRONTEND_COMPOSE:-}" && -f "${LOCAL_FRONTEND_COMPOSE}" ]]; then
|
||||
@@ -239,6 +250,9 @@ run_generate_compose() {
|
||||
)
|
||||
if ${need_backend}; then
|
||||
gen_args+=(--backend-compose "${tmp}/backend.compose.yml")
|
||||
if ${need_caddy}; then
|
||||
gen_args+=(--backend-compose-prod "${tmp}/backend.compose.prod.yml")
|
||||
fi
|
||||
fi
|
||||
if ${need_frontend}; then
|
||||
gen_args+=(--frontend-compose "${tmp}/frontend.compose.yml")
|
||||
|
||||
@@ -123,16 +123,21 @@ def sync_published_compose(
|
||||
tmp = Path(tempfile.mkdtemp(prefix="fromchat-sync-compose-"))
|
||||
try:
|
||||
backend_compose = tmp / "backend.compose.yml"
|
||||
backend_compose_prod = tmp / "backend.compose.prod.yml"
|
||||
web_compose = tmp / "frontend.compose.yml"
|
||||
backend_compose.write_bytes(
|
||||
fetch_raw_file(backend_repo, tag, "compose.yml", token)
|
||||
)
|
||||
backend_compose_prod.write_bytes(
|
||||
fetch_raw_file(backend_repo, tag, "compose.prod.yml", token)
|
||||
)
|
||||
web_compose.write_bytes(
|
||||
fetch_raw_file(web_repo, tag, "compose.yml", token)
|
||||
)
|
||||
|
||||
generate(
|
||||
backend_compose=backend_compose,
|
||||
backend_compose_prod=backend_compose_prod,
|
||||
frontend_compose=web_compose,
|
||||
tag=tag,
|
||||
components=set(components),
|
||||
|
||||
Reference in New Issue
Block a user