commit 50ea5654333533d78646beb400f720f8bf947df9 Author: denis0001-dev Date: Tue Jul 14 12:10:22 2026 +0300 Initial commit diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..e652fdb --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +.env +.deploy-cache/ diff --git a/README.md b/README.md new file mode 100644 index 0000000..a7c837c --- /dev/null +++ b/README.md @@ -0,0 +1,99 @@ +# FromChat Deployment + +One-click installer for a production FromChat stack on Debian-based Linux. + +## Quick install + +```bash +curl -fsSL https://raw.githubusercontent.com/fromchat-messenger/app/main/deployment/install.sh | sudo bash +``` + +The installer: + +1. Ensures root (re-runs via `sudo` if needed) +2. Verifies Debian/Ubuntu +3. Offers to install Docker (official script) +4. Creates `~/fromchat-server` +5. Lets you pick components: **backend**, **frontend**, **caddy**, **updater** +6. Optionally uses custom backend/web git URLs +7. Resolves the latest common semver tag (`v1.0`, `v1.0.0`, …) +8. Downloads only `compose.yml` from each repo at that tag +9. Merges compose files and replaces `build:` with `image: fromchat/:` +10. Opens `Caddyfile` in nano when caddy is selected +11. Runs `docker compose up -d --wait` +12. Optionally starts the [updater](../updater) service + +## Classic deploy (offline, build on your PC) + +For developers who build images **locally** and transfer them to a server via SSH + `docker pussh` — no registry pulls on the server. + +```bash +cd deployment +python3 -m venv .venv && .venv/bin/pip install -r requirements-deploy.txt +# or reuse ../backend/.venv which already has rich + dotenv + +./deploy.sh user@host ~/fromchat-server linux/arm64 --tag latest +``` + +Without `--components`, an interactive checklist opens (backend, frontend, caddy, updater). Non-interactive: + +```bash +./deploy.sh user@host ~/fromchat-server linux/arm64 \ + --components backend,frontend,caddy,updater \ + --tag latest +``` + +This: + +1. Builds all `fromchat/*` images from local `../backend`, `../Web`, `../updater`, and `backend/src/caddy` +2. Merges compose the same way as the installer (`generate-compose.py`) +3. Pulls any third-party images from the registry **on this machine**, then transfers everything via pussh +4. Syncs config to the server and restarts the `fromchat` systemd unit +5. Optionally installs the updater (also built locally) + +The server never pulls from a registry during deploy. Your PC needs network access to pull base images (LiveKit, Docker `FROM` layers, unregistry for pussh). + +Override source trees with `FROMCHAT_BACKEND_DIR`, `FROMCHAT_WEB_DIR`, `FROMCHAT_UPDATER_DIR`. +`DEPLOYMENT_SERVER` in the backend (or deployment) `.env` still works. + +The old entry point `backend/scripts/deploy.sh` redirects here. + +## Generate compose only + +```bash +./install.sh --generate-config backend,frontend,caddy \ + --tag v1.0.0 \ + --output-dir ~/fromchat-server \ + --backend-repo https://github.com/fromchat-messenger/backend.git \ + --web-repo https://github.com/fromchat-messenger/web.git +``` + +## Layout after install + +``` +~/fromchat-server/ + compose.yml # merged production stack + .fromchat-version # current release tag + .env # secrets (create via backend generate:env) + Caddyfile # when caddy selected + config/livekit.yaml + updater/ # when updater selected + compose.yml + .env +``` + +## GitHub token (updater) + +When **updater** is selected, create a token with `read:packages` and `repo` (works for GitHub and git.fromchat.ru): + +https://github.com/settings/tokens/new?description=FromChat%20Updater&scopes=read:packages,repo + +Official `github.com/fromchat-messenger/*` repositories automatically fall back to **git.fromchat.ru/FromChat/** if GitHub is unreachable. + +## Environment overrides + +| Variable | Default | +|----------|---------| +| `FROMCHAT_BACKEND_REPO` | `https://github.com/fromchat-messenger/backend.git` | +| `FROMCHAT_WEB_REPO` | `https://github.com/fromchat-messenger/web.git` | +| `FROMCHAT_APP_REPO` | `https://github.com/fromchat-messenger/app.git` (deployment + updater tooling) | diff --git a/compose/caddy.compose.yml b/compose/caddy.compose.yml new file mode 100644 index 0000000..44dc504 --- /dev/null +++ b/compose/caddy.compose.yml @@ -0,0 +1,15 @@ +services: + caddy: + build: + context: . + dockerfile: Dockerfile + ports: + - "80:80" + - "443:443" + volumes: + - ./Caddyfile:/etc/caddy/Caddyfile:ro + networks: + - public + depends_on: + - web + restart: unless-stopped diff --git a/config/livekit.yaml b/config/livekit.yaml new file mode 100644 index 0000000..c520c57 --- /dev/null +++ b/config/livekit.yaml @@ -0,0 +1,11 @@ +# LiveKit SFU — keys must match LIVEKIT_API_KEY / LIVEKIT_API_SECRET in .env +port: 8303 + +rtc: + tcp_port: 8304 + port_range_start: 50000 + port_range_end: 50100 + use_external_ip: false + +keys: + fromchat_dev: "local_dev_secret_must_be_at_least_32_characters_long" diff --git a/deploy.sh b/deploy.sh new file mode 100755 index 0000000..ae36005 --- /dev/null +++ b/deploy.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash +# Classic FromChat deploy: build local images, generate production compose, push to server. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +DEPLOYMENT_ROOT="${SCRIPT_DIR}" +SCRIPTS="${DEPLOYMENT_ROOT}/scripts" + +# Prefer backend venv (has rich + dotenv), then deployment .venv, then python3. +PYTHON="" +for candidate in \ + "${DEPLOYMENT_ROOT}/../backend/.venv/bin/python3" \ + "${DEPLOYMENT_ROOT}/.venv/bin/python3" \ + "$(command -v python3 || true)" +do + if [[ -n "${candidate}" && -x "${candidate}" ]]; then + PYTHON="${candidate}" + break + fi +done +[[ -n "${PYTHON}" ]] || { echo "python3 not found" >&2; exit 1; } + +export PYTHONPATH="${SCRIPTS}${PYTHONPATH:+:${PYTHONPATH}}" +exec "${PYTHON}" -m deploy.main "$@" diff --git a/install.sh b/install.sh new file mode 100755 index 0000000..661b1b3 --- /dev/null +++ b/install.sh @@ -0,0 +1,254 @@ +#!/usr/bin/env bash +# FromChat one-click server installer +# https://github.com/fromchat-messenger/app +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +DEPLOYMENT_ROOT="${SCRIPT_DIR}" +# shellcheck source=scripts/lib.sh +source "${DEPLOYMENT_ROOT}/scripts/lib.sh" + +DEFAULT_BACKEND_REPO="${FROMCHAT_BACKEND_REPO:-https://github.com/fromchat-messenger/backend.git}" +DEFAULT_WEB_REPO="${FROMCHAT_WEB_REPO:-https://github.com/fromchat-messenger/web.git}" +DEFAULT_APP_REPO="${FROMCHAT_APP_REPO:-https://github.com/fromchat-messenger/app.git}" + +INSTALL_DIR="" +FROMCHAT_VERSION="" +BACKEND_REPO="${DEFAULT_BACKEND_REPO}" +WEB_REPO="${DEFAULT_WEB_REPO}" +GENERATE_ONLY=false +GENERATE_COMPONENTS="" +GENERATE_TAG="" +GENERATE_OUTPUT="" + +usage() { + cat </dev/null | cut -d: -f6 || true)" + if [[ -z "${home}" ]]; then + home="${HOME}" + fi + echo "${home}/fromchat-server" +} + +configure_caddyfile() { + local install_dir="$1" + local caddyfile="${install_dir}/Caddyfile" + if [[ ! -f "${caddyfile}" ]]; then + cp "${DEPLOYMENT_ROOT}/templates/Caddyfile" "${caddyfile}" + fi + step "Opening Caddyfile in nano (save with Ctrl+O, exit with Ctrl+X)…" + "${EDITOR:-nano}" "${caddyfile}" || nano "${caddyfile}" || vi "${caddyfile}" +} + +setup_updater() { + local install_dir="$1" + local tag="$2" + local updater_dir="${install_dir}/updater" + mkdir -p "${updater_dir}" + + step "Git token for the auto-updater (GitHub or Gitea: read packages + repo metadata)." + info "GitHub token:" + printf '%b%s%b\n' "$BLUE" \ + " https://github.com/settings/tokens/new?description=FromChat%20Updater&scopes=read:packages,repo" \ + "$NC" + info "Gitea token: Settings → Applications → Generate New Token (read:repository, read:package)" + prompt "Paste token (input hidden):" + local token + IFS= read -rs token || true + printf '\n' >&2 + [[ -n "${token}" ]] || die "GitHub token is required when updater is selected." + + fetch_raw_file "${DEFAULT_APP_REPO}" "${tag}" "updater/compose.yml" \ + "${updater_dir}/compose.yml" 2>/dev/null \ + || fetch_raw_file "${DEFAULT_APP_REPO}" "main" "updater/compose.yml" \ + "${updater_dir}/compose.yml" 2>/dev/null \ + || cp "${DEPLOYMENT_ROOT}/../updater/compose.yml" "${updater_dir}/compose.yml" + + local components_csv + components_csv="$(IFS=,; echo "${SELECTED[*]}")" + + cat > "${updater_dir}/.env" </dev/null || true + + SELECTED=() + select_components + + prompt "Use custom git repositories? [y/N]:" + local custom + IFS= read -r custom || true + if [[ "${custom:-}" =~ ^[Yy] ]]; then + prompt "Backend repository URL [${DEFAULT_BACKEND_REPO}]:" + IFS= read -r line || true + [[ -n "${line}" ]] && BACKEND_REPO="${line}" + prompt "Web repository URL [${DEFAULT_WEB_REPO}]:" + IFS= read -r line || true + [[ -n "${line}" ]] && WEB_REPO="${line}" + fi + + step "Resolving latest common semver tag…" + local repos_for_tag=() + component_selected backend && repos_for_tag+=("${BACKEND_REPO}") + component_selected frontend && repos_for_tag+=("${WEB_REPO}") + if ((${#repos_for_tag[@]} == 0)); then + repos_for_tag+=("${BACKEND_REPO}" "${WEB_REPO}") + fi + + step "GitHub / Gitea token for private repos (leave empty if public):" + prompt "Paste token (input hidden):" + local gh_token="" + IFS= read -rs gh_token || true + printf '\n' >&2 + export GITHUB_TOKEN="${gh_token}" + export GIT_TOKEN="${gh_token}" + + if [[ -n "${GITHUB_TOKEN}" ]]; then + FROMCHAT_VERSION="$(python3 "${DEPLOYMENT_ROOT}/scripts/resolve-tag.py" "${GITHUB_TOKEN}" "${repos_for_tag[@]}")" + else + FROMCHAT_VERSION="$(python3 "${DEPLOYMENT_ROOT}/scripts/resolve-tag.py" "${repos_for_tag[@]}")" + fi + [[ -n "${FROMCHAT_VERSION}" ]] || die "Could not resolve a semver tag." + success "Using tag ${FROMCHAT_VERSION}" + + local components_csv + components_csv="$(IFS=,; echo "${SELECTED[*]}")" + local compose_components="" + local c + for c in "${SELECTED[@]}"; do + [[ "${c}" == "updater" ]] && continue + compose_components="${compose_components:+$compose_components,}${c}" + done + + run_generate_compose "${INSTALL_DIR}" "${compose_components}" "${FROMCHAT_VERSION}" + + if component_selected caddy; then + configure_caddyfile "${INSTALL_DIR}" + fi + + if [[ ! -f "${INSTALL_DIR}/.env" ]]; then + warn "No .env in ${INSTALL_DIR}. Create one before production use (see backend generate:env)." + touch "${INSTALL_DIR}/.env" + chown "${real_user}:${real_user}" "${INSTALL_DIR}/.env" 2>/dev/null || true + fi + + step "Starting FromChat stack…" + ( + cd "${INSTALL_DIR}" + docker compose --env-file .env up -d --wait --timeout 600 + ) + + if component_selected updater; then + export GITHUB_TOKEN="${gh_token:-${GITHUB_TOKEN:-}}" + export GIT_TOKEN="${gh_token:-${GIT_TOKEN:-${GITHUB_TOKEN:-}}}" + setup_updater "${INSTALL_DIR}" "${FROMCHAT_VERSION}" + fi + + chown -R "${real_user}:${real_user}" "${INSTALL_DIR}" 2>/dev/null || true + print_success_banner "${INSTALL_DIR}" +} + +main() { + parse_args "$@" + if ${GENERATE_ONLY}; then + generate_config_mode + else + full_install "$@" + fi +} + +main "$@" diff --git a/requirements-deploy.txt b/requirements-deploy.txt new file mode 100644 index 0000000..3c21163 --- /dev/null +++ b/requirements-deploy.txt @@ -0,0 +1,3 @@ +rich>=13.9.4 +python-dotenv>=1.0.1 +PyYAML>=6.0.1 diff --git a/scripts/__pycache__/git_remote.cpython-312.pyc b/scripts/__pycache__/git_remote.cpython-312.pyc new file mode 100644 index 0000000..c393adc Binary files /dev/null and b/scripts/__pycache__/git_remote.cpython-312.pyc differ diff --git a/scripts/deploy/__init__.py b/scripts/deploy/__init__.py new file mode 100644 index 0000000..7783d98 --- /dev/null +++ b/scripts/deploy/__init__.py @@ -0,0 +1 @@ +"""FromChat deployment orchestration (Docker build, pussh, rsync, remote systemd).""" diff --git a/scripts/deploy/__pycache__/__init__.cpython-312.pyc b/scripts/deploy/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000..03075e0 Binary files /dev/null and b/scripts/deploy/__pycache__/__init__.cpython-312.pyc differ diff --git a/scripts/deploy/__pycache__/components.cpython-312.pyc b/scripts/deploy/__pycache__/components.cpython-312.pyc new file mode 100644 index 0000000..5dba5a6 Binary files /dev/null and b/scripts/deploy/__pycache__/components.cpython-312.pyc differ diff --git a/scripts/deploy/__pycache__/compose_build.cpython-312.pyc b/scripts/deploy/__pycache__/compose_build.cpython-312.pyc new file mode 100644 index 0000000..6e58a96 Binary files /dev/null and b/scripts/deploy/__pycache__/compose_build.cpython-312.pyc differ diff --git a/scripts/deploy/__pycache__/config.cpython-312.pyc b/scripts/deploy/__pycache__/config.cpython-312.pyc new file mode 100644 index 0000000..ea74726 Binary files /dev/null and b/scripts/deploy/__pycache__/config.cpython-312.pyc differ diff --git a/scripts/deploy/__pycache__/docker_local.cpython-312.pyc b/scripts/deploy/__pycache__/docker_local.cpython-312.pyc new file mode 100644 index 0000000..81f0575 Binary files /dev/null and b/scripts/deploy/__pycache__/docker_local.cpython-312.pyc differ diff --git a/scripts/deploy/__pycache__/main.cpython-312.pyc b/scripts/deploy/__pycache__/main.cpython-312.pyc new file mode 100644 index 0000000..a79239c Binary files /dev/null and b/scripts/deploy/__pycache__/main.cpython-312.pyc differ diff --git a/scripts/deploy/__pycache__/paths.cpython-312.pyc b/scripts/deploy/__pycache__/paths.cpython-312.pyc new file mode 100644 index 0000000..c42287c Binary files /dev/null and b/scripts/deploy/__pycache__/paths.cpython-312.pyc differ diff --git a/scripts/deploy/__pycache__/ssh_auth.cpython-312.pyc b/scripts/deploy/__pycache__/ssh_auth.cpython-312.pyc new file mode 100644 index 0000000..4ec335d Binary files /dev/null and b/scripts/deploy/__pycache__/ssh_auth.cpython-312.pyc differ diff --git a/scripts/deploy/__pycache__/transfer.cpython-312.pyc b/scripts/deploy/__pycache__/transfer.cpython-312.pyc new file mode 100644 index 0000000..28e337c Binary files /dev/null and b/scripts/deploy/__pycache__/transfer.cpython-312.pyc differ diff --git a/scripts/deploy/__pycache__/ui.cpython-312.pyc b/scripts/deploy/__pycache__/ui.cpython-312.pyc new file mode 100644 index 0000000..bd98658 Binary files /dev/null and b/scripts/deploy/__pycache__/ui.cpython-312.pyc differ diff --git a/scripts/deploy/__pycache__/updater.cpython-312.pyc b/scripts/deploy/__pycache__/updater.cpython-312.pyc new file mode 100644 index 0000000..9531dfe Binary files /dev/null and b/scripts/deploy/__pycache__/updater.cpython-312.pyc differ diff --git a/scripts/deploy/__pycache__/util.cpython-312.pyc b/scripts/deploy/__pycache__/util.cpython-312.pyc new file mode 100644 index 0000000..ac7bdfd Binary files /dev/null and b/scripts/deploy/__pycache__/util.cpython-312.pyc differ diff --git a/scripts/deploy/components.py b/scripts/deploy/components.py new file mode 100644 index 0000000..d24782c --- /dev/null +++ b/scripts/deploy/components.py @@ -0,0 +1,95 @@ +"""Interactive component selection (matches deployment/scripts/lib.sh).""" + +from __future__ import annotations + +import shutil +import subprocess +import sys + +import deploy.ui as ui + +COMPONENT_OPTIONS: list[tuple[str, str, bool]] = [ + ("backend", "Backend (API, DB, LiveKit)", True), + ("frontend", "Web frontend", True), + ("caddy", "Caddy reverse proxy (TLS)", False), + ("updater", "Auto-update service", False), +] + +ALLOWED = {name for name, _, _ in COMPONENT_OPTIONS} + + +def parse_components_csv(raw: str) -> list[str]: + out: list[str] = [] + for part in raw.split(","): + c = part.strip().lower() + if not c: + continue + if c not in ALLOWED: + sys.stderr.write( + f"Unknown component: {c} (allowed: {','.join(sorted(ALLOWED))})\n" + ) + raise SystemExit(1) + if c not in out: + out.append(c) + if not out: + sys.stderr.write("Select at least one component.\n") + raise SystemExit(1) + return out + + +def _select_whiptail() -> list[str]: + args = [ + "whiptail", + "--title", + "FromChat components", + "--checklist", + "Select components (Space toggles, Enter confirms)", + "18", + "72", + str(len(COMPONENT_OPTIONS)), + ] + for name, label, default_on in COMPONENT_OPTIONS: + args.extend([name, label, "ON" if default_on else "OFF"]) + result = subprocess.run(args, capture_output=True, text=True) + if result.returncode != 0: + ui.error("Component selection cancelled.") + raise SystemExit(1) + selected: list[str] = [] + for token in result.stdout.split(): + name = token.strip('"') + if name in ALLOWED: + selected.append(name) + if not selected: + ui.error("Select at least one component.") + raise SystemExit(1) + return selected + + +def _select_text() -> list[str]: + ui.warning("whiptail not found; using text menu.") + selected: list[str] = [] + for name, label, default_on in COMPONENT_OPTIONS: + hint = "Y/n" if default_on else "y/N" + answer = input(f" Include {label}? [{hint}]: ").strip() + if not answer: + if default_on: + selected.append(name) + continue + if answer.lower().startswith("y"): + selected.append(name) + if not selected: + ui.error("Select at least one component.") + raise SystemExit(1) + return selected + + +def select_components_interactive() -> list[str]: + ui.step("Select components") + if shutil.which("whiptail"): + return _select_whiptail() + return _select_text() + + +def compose_components(selected: list[str]) -> list[str]: + """Components that go into the merged stack compose (excludes updater).""" + return [c for c in selected if c != "updater"] diff --git a/scripts/deploy/compose_build.py b/scripts/deploy/compose_build.py new file mode 100644 index 0000000..6d993b3 --- /dev/null +++ b/scripts/deploy/compose_build.py @@ -0,0 +1,341 @@ +"""Parse docker-compose JSON and run image builds for selected components.""" + +from __future__ import annotations + +import json +import os +import subprocess +import sys +from dataclasses import dataclass +from pathlib import Path + +from deploy.paths import ProjectPaths +import deploy.ui as ui +from deploy.util import ( + compute_inputs_hash, + dedupe_preserve, + local_docker_image_tags, + local_image_layer_fp, + read_file_if_exists, + sanitize_ref, +) + +FROMCHAT_IMAGE_SERVICES = frozenset( + {"main", "messaging", "file_storage", "postgres", "web", "caddy", "updater"} +) +FROMCHAT_PREFIX = "fromchat/" + + +@dataclass +class PushableService: + service: str + image_tag: str + dockerfile: Path + build_context: Path + build_target: str + input_hash: str + compose_root: Path + + +def fromchat_image(service: str, tag: str) -> str: + return f"{FROMCHAT_PREFIX}{service}:{tag}" + + +class ComposeBuildPhase: + def __init__( + self, + paths: ProjectPaths, + *, + tag: str, + platform: str, + use_docker_build: bool, + ) -> None: + self._paths = paths + self._tag = tag + self._platform = platform + self._use_docker_build = use_docker_build + + def load_compose_json(self, compose_root: Path) -> dict: + env = os.environ.copy() + env["COMPOSE_PROFILES"] = "production" + p = subprocess.run( + ["docker", "compose", "-f", "compose.yml", "config", "--format", "json"], + cwd=compose_root, + capture_output=True, + text=True, + env=env, + ) + if p.returncode != 0: + ui.error(f"docker compose config failed in {compose_root}") + if p.stderr: + print(p.stderr, file=sys.stderr) + sys.exit(1) + return json.loads(p.stdout) + + def list_services(self, compose_root: Path) -> list[str]: + env = os.environ.copy() + env["COMPOSE_PROFILES"] = "production" + p = subprocess.run( + ["docker", "compose", "-f", "compose.yml", "config", "--services"], + cwd=compose_root, + capture_output=True, + text=True, + env=env, + ) + if p.returncode != 0: + return [] + return [s.strip() for s in p.stdout.splitlines() if s.strip()] + + def _make_pushable( + self, + *, + service: str, + image_tag: str, + dockerfile: Path, + build_context: Path, + build_target: str, + compose_root: Path, + ) -> PushableService | None: + if not dockerfile.is_file(): + ui.error(f"Could not find Dockerfile for {service} at {dockerfile}") + sys.exit(1) + if not self._paths.input_hash_script.is_file(): + ui.error(f"Missing {self._paths.input_hash_script}") + sys.exit(1) + h = compute_inputs_hash( + build_context, + dockerfile, + hash_script=self._paths.input_hash_script, + ) + if not h: + ui.error(f"Failed to compute input hash for {service}") + sys.exit(1) + return PushableService( + service=service, + image_tag=image_tag, + dockerfile=dockerfile, + build_context=build_context, + build_target=build_target, + input_hash=h, + compose_root=compose_root, + ) + + def collect_pushable( + self, + compose: dict, + services: list[str], + compose_root: Path, + ) -> list[PushableService]: + out: list[PushableService] = [] + svc_map = compose.get("services") or {} + for service in services: + if service not in FROMCHAT_IMAGE_SERVICES: + continue + spec = svc_map.get(service) + if not isinstance(spec, dict): + continue + build = spec.get("build") + if not isinstance(build, dict): + continue + image_tag = fromchat_image(service, self._tag) + dockerfile_rel = (build.get("dockerfile") or "").strip() + context_rel = (build.get("context") or ".").strip() + build_target = (build.get("target") or "").strip() + if context_rel in (".", "./", ""): + build_context = compose_root + elif context_rel == "..": + build_context = compose_root.parent + elif context_rel.startswith("/"): + build_context = Path(context_rel) + else: + build_context = (compose_root / context_rel).resolve() + if dockerfile_rel: + if dockerfile_rel.startswith("/"): + dockerfile = Path(dockerfile_rel) + elif (compose_root / dockerfile_rel).is_file(): + dockerfile = compose_root / dockerfile_rel + else: + dockerfile = build_context / dockerfile_rel + else: + dockerfile = build_context / "Dockerfile" + ps = self._make_pushable( + service=service, + image_tag=image_tag, + dockerfile=dockerfile, + build_context=build_context, + build_target=build_target, + compose_root=compose_root, + ) + if ps: + out.append(ps) + return out + + def collect_caddy_pushable(self) -> PushableService | None: + caddy_dir = self._paths.caddy_build_dir + if not caddy_dir: + ui.error( + "Caddy selected but backend/src/caddy not found. " + "Set FROMCHAT_BACKEND_DIR to a backend checkout." + ) + sys.exit(1) + return self._make_pushable( + service="caddy", + image_tag=fromchat_image("caddy", self._tag), + dockerfile=caddy_dir / "Dockerfile", + build_context=caddy_dir, + build_target="", + compose_root=caddy_dir, + ) + + def collect_updater_pushable(self) -> PushableService | None: + updater_dir = self._paths.updater_dir + if not updater_dir: + ui.error( + "Updater selected but ../updater not found. Set FROMCHAT_UPDATER_DIR." + ) + sys.exit(1) + dockerfile = updater_dir / "Dockerfile" + return self._make_pushable( + service="updater", + image_tag=fromchat_image("updater", "latest"), + dockerfile=dockerfile, + build_context=updater_dir, + build_target="", + compose_root=updater_dir, + ) + + def plan_builds(self, pushable: list[PushableService]) -> list[PushableService]: + cache_root = self._paths.local_image_cache_dir + cache_root.mkdir(parents=True, exist_ok=True) + to_build: list[PushableService] = [] + for ps in pushable: + key = sanitize_ref(ps.image_tag) + cache_file = cache_root / key / "input.sha256" + prev = read_file_if_exists(cache_file).strip() + fp = local_image_layer_fp(ps.image_tag) + if prev and prev == ps.input_hash and fp: + continue + to_build.append(ps) + return to_build + + def run_builds(self, to_build: list[PushableService]) -> list[str]: + if not to_build: + ui.success("Build skipped (no Docker inputs changed)") + return [] + ui.step(f"Building {len(to_build)} image(s) locally") + for ps in to_build: + ui.substep(f"Building {ps.service} -> {ps.image_tag}...") + if self._use_docker_build: + args = [ + "docker", + "build", + "--pull", + "--file", + str(ps.dockerfile), + "--tag", + ps.image_tag, + ] + else: + args = [ + "docker", + "buildx", + "build", + "--pull", + "--platform", + self._platform, + "--file", + str(ps.dockerfile), + "--tag", + ps.image_tag, + "--output=type=docker", + "--provenance=false", + "--sbom=false", + ] + if ps.build_target: + args.extend(["--target", ps.build_target]) + args.append(str(ps.build_context)) + if subprocess.run(args, cwd=ps.compose_root).returncode != 0: + ui.error(f"Build failed for {ps.service}") + sys.exit(1) + built: list[str] = [] + for ps in to_build: + key = sanitize_ref(ps.image_tag) + d = self._paths.local_image_cache_dir / key + d.mkdir(parents=True, exist_ok=True) + (d / "input.sha256").write_text(ps.input_hash, encoding="utf-8") + built.append(ps.image_tag) + ui.success(f"Build complete! {len(built)} image(s) built") + return built + + +def classify_compose_images( + compose: dict, + service_order: list[str], +) -> tuple[list[str], list[str]]: + """Return (fromchat images to transfer, third-party images to transfer).""" + services = compose.get("services") or {} + fromchat_images: list[str] = [] + third_party: list[str] = [] + for name in service_order: + spec = services.get(name) + if not isinstance(spec, dict): + continue + image_from = (spec.get("image") or "").strip() + if not image_from: + continue + if image_from.startswith(FROMCHAT_PREFIX): + fromchat_images.append(image_from) + else: + third_party.append(image_from) + return dedupe_preserve(fromchat_images), dedupe_preserve(third_party) + + +def images_present_locally(images: list[str], local_tags: set[str]) -> list[str]: + return dedupe_preserve([img for img in images if img in local_tags]) + + +def verify_fromchat_images_local(fromchat_images: list[str], local_tags: set[str], ui_mod: object) -> None: + missing = [img for img in fromchat_images if img not in local_tags] + if missing: + ui_mod.error( + "Missing locally built images: " + ", ".join(missing) + ) + print("Build them on this machine first, then deploy.", file=sys.stderr) + sys.exit(1) + + +def pull_third_party_images(images: list[str], *, platform: str | None = None) -> None: + if not images: + return + import deploy.docker_local as docker_local + + docker_local.pull_images(images, platform=platform) + + +def generate_production_compose( + paths: ProjectPaths, + *, + components: list[str], + tag: str, + output: Path, +) -> None: + ui.step("Generating production compose.yml") + cmd = [ + sys.executable, + str(paths.generate_compose_script), + "--tag", + tag, + "--components", + ",".join(components), + "--output", + str(output), + ] + if "backend" in components and paths.backend_dir: + cmd.extend(["--backend-compose", str(paths.backend_dir / "compose.yml")]) + if "frontend" in components and paths.web_dir: + cmd.extend(["--frontend-compose", str(paths.web_dir / "compose.yml")]) + if "caddy" in components: + cmd.extend(["--caddy-compose", str(paths.caddy_compose)]) + if subprocess.run(cmd).returncode != 0: + ui.error("generate-compose.py failed") + sys.exit(1) diff --git a/scripts/deploy/config.py b/scripts/deploy/config.py new file mode 100644 index 0000000..4947d10 --- /dev/null +++ b/scripts/deploy/config.py @@ -0,0 +1,158 @@ +"""Load .env and CLI into settings.""" + +from __future__ import annotations + +import os +import platform +import sys +from dataclasses import dataclass + +from dotenv import load_dotenv + +from deploy.components import compose_components, parse_components_csv, select_components_interactive +from deploy.paths import ProjectPaths + + +@dataclass +class DeploySettings: + server: str + deploy_path: str + platform: str + host_arch: str + platform_arch: str + use_docker_build: bool + components: list[str] + compose_components: list[str] + tag: str + git_token: str | None + paths: ProjectPaths + + +def _machine_arch() -> str: + m = platform.machine().lower() + if m in ("arm64", "aarch64"): + return "arm64" + if m in ("x86_64", "amd64", "i386", "i686"): + return "amd64" + return m + + +def load_settings(paths: ProjectPaths, argv: list[str]) -> DeploySettings: + if paths.env_file.is_file(): + load_dotenv(paths.env_file, override=False) + + positional: list[str] = [] + components_raw: str | None = os.environ.get("FROMCHAT_COMPONENTS") + tag = os.environ.get("FROMCHAT_TAG", "latest") + git_token: str | None = os.environ.get("GIT_TOKEN") or os.environ.get("GITHUB_TOKEN") + interactive = True + i = 1 + while i < len(argv): + arg = argv[i] + if arg in ("--components", "-c"): + if i + 1 >= len(argv): + sys.stderr.write("--components requires a value\n") + raise SystemExit(1) + components_raw = argv[i + 1] + interactive = False + i += 2 + continue + if arg in ("--tag", "-t"): + if i + 1 >= len(argv): + sys.stderr.write("--tag requires a value\n") + raise SystemExit(1) + tag = argv[i + 1] + i += 2 + continue + if arg in ("--no-interactive",): + interactive = False + i += 1 + continue + if arg in ("-h", "--help"): + sys.stdout.write( + "Usage: deploy.sh [user@host] [deploy_path] [platform] " + "[--components backend,frontend,caddy,updater] [--tag TAG]\n" + " Interactive component menu when --components is omitted.\n" + " Or set DEPLOYMENT_SERVER in .env\n" + " Paths: FROMCHAT_BACKEND_DIR, FROMCHAT_WEB_DIR, FROMCHAT_UPDATER_DIR\n" + ) + raise SystemExit(0) + if arg.startswith("-"): + sys.stderr.write(f"Unknown option: {arg}\n") + raise SystemExit(1) + positional.append(arg) + i += 1 + + server = (positional[0] if positional else None) or os.environ.get("DEPLOYMENT_SERVER", "") + server = server.strip() + if not server: + sys.stderr.write( + "Server not specified. Usage: deploy.sh [user@host] [deploy_path] [platform]\n" + f" Or set DEPLOYMENT_SERVER in {paths.env_file} or as an environment variable\n\n" + "Example:\n" + " deploy.sh user@example.com ~/fromchat-server linux/arm64 --tag latest\n" + ) + raise SystemExit(1) + + deploy_path = ( + (positional[1] if len(positional) > 1 else None) + or os.environ.get("DEPLOYMENT_PATH", "") + or "~/fromchat-server" + ).strip() + docker_platform = ( + (positional[2] if len(positional) > 2 else None) + or os.environ.get("DEPLOYMENT_PLATFORM", "") + or "linux/arm64" + ).strip() + + if components_raw is not None: + components = parse_components_csv(components_raw) + elif interactive and sys.stdin.isatty(): + components = select_components_interactive() + else: + components = parse_components_csv("backend,frontend") + + stack = compose_components(components) + + host_arch = _machine_arch() + platform_arch = docker_platform.split("/", 1)[-1] + use_docker_build = bool(host_arch and host_arch == platform_arch) + + if "backend" in components and not paths.backend_dir: + sys.stderr.write( + "Backend component selected but backend repo not found.\n" + "Set FROMCHAT_BACKEND_DIR or keep a sibling ../backend with compose.yml.\n" + ) + raise SystemExit(1) + if "frontend" in components and not paths.web_dir: + sys.stderr.write( + "Frontend component selected but web repo not found.\n" + "Set FROMCHAT_WEB_DIR or keep a sibling ../Web with compose.yml.\n" + ) + raise SystemExit(1) + if "caddy" in components and not paths.caddy_build_dir: + sys.stderr.write( + "Caddy component selected but backend/src/caddy not found.\n" + "Set FROMCHAT_BACKEND_DIR to a backend checkout.\n" + ) + raise SystemExit(1) + if "updater" in components and not paths.updater_dir: + sys.stderr.write( + "Updater component selected but ../updater not found.\n" + "Set FROMCHAT_UPDATER_DIR to the updater repo.\n" + ) + raise SystemExit(1) + + return DeploySettings( + server=server, + deploy_path=deploy_path, + platform=docker_platform, + host_arch=host_arch, + platform_arch=platform_arch, + use_docker_build=use_docker_build, + components=components, + compose_components=stack, + tag=tag, + git_token=git_token, + paths=paths, + ) diff --git a/scripts/deploy/docker_local.py b/scripts/deploy/docker_local.py new file mode 100644 index 0000000..8922041 --- /dev/null +++ b/scripts/deploy/docker_local.py @@ -0,0 +1,108 @@ +"""Local Docker daemon, Docker Desktop, and buildx setup.""" + +from __future__ import annotations + +import subprocess +import sys +import time + +import deploy.ui as ui + + +BUILDER_NAME = "fromchat-builder" + + +def ensure_daemon() -> None: + if _daemon_ok(): + return + ui.warning("Docker daemon is not running") + if not _start_desktop(): + ui.error("Failed to start Docker Desktop. Please start it manually and try again.") + sys.exit(1) + + +def _daemon_ok() -> bool: + return subprocess.run(["docker", "info"], capture_output=True).returncode == 0 + + +def _start_desktop() -> bool: + ui.substep("Starting Docker Desktop...") + if subprocess.run(["docker", "desktop", "start"], capture_output=True).returncode != 0: + return False + ui.substep("Waiting for Docker to start...", end="") + sys.stdout.flush() + max_wait = 60 + waited = 0 + while waited < max_wait: + if _daemon_ok(): + print() + return True + time.sleep(2) + waited += 2 + print(".", end="", flush=True) + print() + return False + + +def ensure_buildx(use_compose_build: bool) -> None: + if use_compose_build: + return + if subprocess.run(["docker", "buildx", "version"], capture_output=True).returncode != 0: + ui.error("Docker buildx not available. Install Docker Desktop.") + sys.exit(1) + _setup_builder() + + +def _setup_builder() -> None: + ui.step("Setting up buildx builder") + name = BUILDER_NAME + exists = subprocess.run(["docker", "buildx", "inspect", name], capture_output=True).returncode == 0 + if exists: + if subprocess.run(["docker", "buildx", "use", name], capture_output=True).returncode != 0: + ui.substep("Recreating builder...") + subprocess.run(["docker", "buildx", "rm", name], capture_output=True) + exists = False + elif subprocess.run(["docker", "buildx", "inspect", name], capture_output=True).returncode != 0: + ui.substep("Recreating builder (inspection failed)...") + subprocess.run(["docker", "buildx", "rm", name], capture_output=True) + exists = False + if not exists: + ui.substep("Creating builder with persistent cache...") + subprocess.run( + [ + "docker", + "buildx", + "create", + "--name", + name, + "--driver", + "docker-container", + "--driver-opt", + "image=moby/buildkit:latest", + "--use", + "--bootstrap", + ], + capture_output=True, + ) + subprocess.run(["docker", "buildx", "use", name], capture_output=True) + + +def pull_images(images: list[str], *, platform: str | None = None) -> None: + """Pull third-party images on this machine (then transfer via pussh).""" + from deploy.util import dedupe_preserve + + unique = dedupe_preserve([img for img in images if img.strip()]) + if not unique: + return + ui.step(f"Pulling {len(unique)} remote image(s) locally") + for image in unique: + ui.substep(f"Pulling {image}...") + cmd = ["docker", "pull"] + if platform: + cmd.extend(["--platform", platform]) + cmd.append(image) + if subprocess.run(cmd).returncode != 0: + ui.error(f"Failed to pull {image}") + sys.exit(1) + ui.success("Remote images ready locally") + diff --git a/scripts/deploy/main.py b/scripts/deploy/main.py new file mode 100644 index 0000000..5aeef79 --- /dev/null +++ b/scripts/deploy/main.py @@ -0,0 +1,223 @@ +"""CLI entry: build locally, transfer images via pussh, rsync config — no registry pulls.""" + +from __future__ import annotations + +import json +import os +import shutil +import subprocess +import sys +from pathlib import Path + +_SCRIPTS = Path(__file__).resolve().parent.parent +if str(_SCRIPTS) not in sys.path: + sys.path.insert(0, str(_SCRIPTS)) + +from deploy.compose_build import ( # noqa: E402 + ComposeBuildPhase, + classify_compose_images, + fromchat_image, + generate_production_compose, + pull_third_party_images, + verify_fromchat_images_local, +) +from deploy.config import load_settings # noqa: E402 +import deploy.docker_local as docker_local # noqa: E402 +from deploy.paths import ProjectPaths # noqa: E402 +from deploy.ssh_auth import SshAuth # noqa: E402 +from deploy.transfer import DeployTransfer # noqa: E402 +from deploy.updater import setup_updater_remote # noqa: E402 +import deploy.ui as ui # noqa: E402 +from deploy.util import dedupe_preserve, local_docker_image_tags # noqa: E402 + +UPDATER_IMAGE = fromchat_image("updater", "latest") + + +def _sync_caddy_build_files(paths: ProjectPaths, compose_components: list[str]) -> None: + if "caddy" not in compose_components or not paths.caddy_build_dir: + return + if paths.caddyfile_template.is_file(): + shutil.copy2(paths.caddyfile_template, paths.caddy_build_dir / "Caddyfile") + + +def _prepare_staging(paths: ProjectPaths, compose_components: list[str], tag: str) -> Path: + staging = paths.staging_dir + if staging.exists(): + shutil.rmtree(staging) + staging.mkdir(parents=True) + (staging / "config").mkdir() + + if compose_components: + generate_production_compose( + paths, components=compose_components, tag=tag, output=staging / "compose.yml" + ) + (staging / ".fromchat-version").write_text(tag + "\n", encoding="utf-8") + + if paths.livekit_config.is_file(): + shutil.copy2(paths.livekit_config, staging / "config" / "livekit.yaml") + + if "caddy" in compose_components: + caddyfile = staging / "Caddyfile" + if paths.caddyfile_template.is_file(): + shutil.copy2(paths.caddyfile_template, caddyfile) + + if paths.systemd_unit_template.is_file(): + unit = paths.systemd_unit_template.read_text(encoding="utf-8") + if "WorkingDirectory=" not in unit: + unit += "\nWorkingDirectory=/opt/fromchat-server\n" + (staging / "fromchat.service").write_text(unit, encoding="utf-8") + + return staging + + +def _load_generated_compose(staging: Path) -> dict: + compose_file = staging / "compose.yml" + if not compose_file.is_file(): + return {} + env = os.environ.copy() + p = subprocess.run( + ["docker", "compose", "-f", "compose.yml", "config", "--format", "json"], + cwd=staging, + capture_output=True, + text=True, + env=env, + ) + if p.returncode != 0: + ui.warning("docker compose config on staging failed; using raw YAML structure") + import yaml + + return yaml.safe_load(compose_file.read_text(encoding="utf-8")) or {} + return json.loads(p.stdout) + + +def _collect_all_pushable( + build_phase: ComposeBuildPhase, + paths: ProjectPaths, + stack: list[str], + include_updater: bool, +) -> list: + pushable = [] + ui.step("Detecting buildable services") + for component in stack: + if component == "caddy": + ps = build_phase.collect_caddy_pushable() + if ps: + pushable.append(ps) + ui.substep(f"caddy: {ps.service}") + continue + compose_root = paths.project_root_for(component) + if not compose_root: + continue + services = build_phase.list_services(compose_root) + if not services: + ui.warning(f"No services in {compose_root}/compose.yml") + continue + compose_json = build_phase.load_compose_json(compose_root) + found = build_phase.collect_pushable(compose_json, services, compose_root) + ui.substep(f"{component}: {', '.join(p.service for p in found) or '(none)'}") + pushable.extend(found) + + if include_updater: + ps = build_phase.collect_updater_pushable() + if ps: + pushable.append(ps) + ui.substep(f"updater: {ps.service}") + + return pushable + + +def main() -> None: + paths = ProjectPaths.from_deploy_package() + settings = load_settings(paths, sys.argv) + ui.banner() + ui.info("Build locally, pull remote images on this PC, transfer via pussh") + ui.info(f"Components: {', '.join(settings.components)}") + ui.info(f"Image tag: {settings.tag}") + if paths.backend_dir: + ui.info(f"Backend: {paths.backend_dir}") + if paths.web_dir: + ui.info(f"Web: {paths.web_dir}") + + creds = SshAuth(settings.server).authenticate() + transfer = DeployTransfer(paths) + deploy_resolved = transfer.resolve_deploy_path_on_server(settings.server, settings.deploy_path) + + stack = settings.compose_components + include_updater = "updater" in settings.components + + docker_local.ensure_daemon() + docker_local.ensure_buildx(settings.use_docker_build) + + build_phase = ComposeBuildPhase( + paths, + tag=settings.tag, + platform=settings.platform, + use_docker_build=settings.use_docker_build, + ) + + all_pushable = _collect_all_pushable( + build_phase, paths, stack, include_updater=include_updater + ) + + if not all_pushable: + if stack or include_updater: + ui.error("No buildable services found for selected components") + raise SystemExit(1) + + _sync_caddy_build_files(paths, stack) + + ui.build_banner() + to_build = build_phase.plan_builds(all_pushable) + build_phase.run_builds(to_build) + + staging = _prepare_staging(paths, stack, settings.tag) + + ui.deploy_banner(settings.server) + transfer.ensure_pussh() + + images_to_transfer: list[str] = [] + + if stack: + generated = _load_generated_compose(staging) + services = list((generated.get("services") or {}).keys()) + local_tags = local_docker_image_tags() + fromchat_images, third_party = classify_compose_images(generated, services) + + verify_fromchat_images_local(fromchat_images, local_tags, ui) + pull_third_party_images(third_party, platform=settings.platform) + + images_to_transfer.extend(fromchat_images) + images_to_transfer.extend(third_party) + + if include_updater: + local_tags = local_docker_image_tags() + verify_fromchat_images_local([UPDATER_IMAGE], local_tags, ui) + images_to_transfer.append(UPDATER_IMAGE) + + images_to_transfer = dedupe_preserve(images_to_transfer) + transfer.pussh_images(creds, images_to_transfer) + + if stack: + transfer.rsync_staging(creds, settings.deploy_path, staging) + transfer.copy_env_prod(creds, settings.deploy_path) + if "backend" in stack: + deploy_resolved = transfer.sync_firebase_cert(creds, settings.deploy_path) + transfer.run_remote_systemd(creds, deploy_resolved) + + if include_updater: + setup_updater_remote( + creds, + settings.deploy_path, + deploy_resolved, + components=settings.components, + paths=paths, + git_token=settings.git_token, + sudo_password=creds.sudo_password, + ) + + print() + ui.success("Deployment complete!") + + +if __name__ == "__main__": + main() diff --git a/scripts/deploy/paths.py b/scripts/deploy/paths.py new file mode 100644 index 0000000..ef35a69 --- /dev/null +++ b/scripts/deploy/paths.py @@ -0,0 +1,118 @@ +"""Resolved paths for classic deploy against sibling backend/web repos.""" + +from __future__ import annotations + +import os +from dataclasses import dataclass +from pathlib import Path + + +def _find_repo_with_compose(*candidates: str) -> Path | None: + for cand in candidates: + if not cand: + continue + p = Path(cand).expanduser().resolve() + if p.is_dir() and (p / "compose.yml").is_file(): + return p + return None + + +def _find_dir(*candidates: str) -> Path | None: + for cand in candidates: + if not cand: + continue + p = Path(cand).expanduser().resolve() + if p.is_dir(): + return p + return None + + +@dataclass(frozen=True) +class ProjectPaths: + """Deployment repo + local component source trees.""" + + deployment_root: Path + scripts_dir: Path + backend_dir: Path | None + web_dir: Path | None + updater_dir: Path | None + caddy_build_dir: Path | None + env_file: Path + staging_dir: Path + local_cache_root: Path + local_image_cache_dir: Path + input_hash_script: Path + livekit_config: Path + caddy_compose: Path + caddyfile_template: Path + systemd_unit_template: Path + generate_compose_script: Path + + @classmethod + def from_deploy_package(cls) -> ProjectPaths: + deploy_pkg = Path(__file__).resolve().parent + scripts_dir = deploy_pkg.parent + deployment_root = scripts_dir.parent + parent = deployment_root.parent + + backend = _find_repo_with_compose( + os.environ.get("FROMCHAT_BACKEND_DIR", ""), + str(parent / "backend"), + str(parent / "Backend"), + ) + web = _find_repo_with_compose( + os.environ.get("FROMCHAT_WEB_DIR", ""), + str(parent / "Web"), + str(parent / "web"), + ) + updater = _find_dir( + os.environ.get("FROMCHAT_UPDATER_DIR", ""), + str(parent / "updater"), + ) + caddy_build = None + if backend: + cand = backend / "src" / "caddy" + if cand.is_dir() and (cand / "Dockerfile").is_file(): + caddy_build = cand + + env_file = deployment_root / ".env" + if backend and (backend / ".env").is_file(): + env_file = backend / ".env" + + cache = deployment_root / ".deploy-cache" + return cls( + deployment_root=deployment_root, + scripts_dir=scripts_dir, + backend_dir=backend, + web_dir=web, + updater_dir=updater, + caddy_build_dir=caddy_build, + env_file=env_file, + staging_dir=cache / "staging", + local_cache_root=cache, + local_image_cache_dir=cache / "images", + input_hash_script=scripts_dir / "docker_inputs_hash.py", + livekit_config=deployment_root / "config" / "livekit.yaml", + caddy_compose=deployment_root / "compose" / "caddy.compose.yml", + caddyfile_template=deployment_root / "templates" / "Caddyfile", + systemd_unit_template=deployment_root / "templates" / "fromchat.service", + generate_compose_script=scripts_dir / "generate-compose.py", + ) + + def compose_for(self, component: str) -> Path | None: + if component == "backend" and self.backend_dir: + return self.backend_dir / "compose.yml" + if component == "frontend" and self.web_dir: + return self.web_dir / "compose.yml" + if component == "caddy": + return self.caddy_compose if self.caddy_compose.is_file() else None + return None + + def project_root_for(self, component: str) -> Path | None: + if component == "backend": + return self.backend_dir + if component == "frontend": + return self.web_dir + if component == "caddy": + return self.caddy_build_dir + return None diff --git a/scripts/deploy/ssh_auth.py b/scripts/deploy/ssh_auth.py new file mode 100644 index 0000000..34d44e6 --- /dev/null +++ b/scripts/deploy/ssh_auth.py @@ -0,0 +1,263 @@ +"""SSH key agent, host trust, pubkey install, and optional sudo password.""" + +from __future__ import annotations + +import getpass +import os +import shutil +import subprocess +import sys +from dataclasses import dataclass +from pathlib import Path + +import deploy.ui as ui + +DEFAULT_KEY_FILE = Path.home() / ".ssh" / "id_rsa" +KEYGEN_DISPLAY = ( + 'ssh-keygen -t rsa -b 4096 -f ~/.ssh/id_rsa -N "" -C "fromchat-deploy"' +) + + +@dataclass +class SshCredentials: + server: str + sudo_password: str + + +def ssh_common_options() -> list[str]: + return [ + "-o", + "StrictHostKeyChecking=accept-new", + "-o", + "ConnectTimeout=10", + ] + + +def ssh_argv(server: str, remote_command: str) -> list[str]: + return ["ssh", *ssh_common_options(), server, remote_command] + + +def scp_argv(local: str, remote: str) -> list[str]: + return ["scp", *ssh_common_options(), local, remote] + + +class SshAuth: + def __init__(self, server: str) -> None: + self._server = server + + def authenticate(self) -> SshCredentials: + ui.step("Authentication") + self._ensure_agent() + key_file = self._resolve_key_file() + self._ensure_key_file(key_file) + self._ensure_key_in_agent(key_file) + self._trust_host_key() + sudo_password = self._read_sudo_password() + self._verify_key_auth(key_file, sudo_password) + sudo_password = self._verify_sudo_password(sudo_password) + return SshCredentials(server=self._server, sudo_password=sudo_password) + + def _read_sudo_password(self) -> str: + pw = getpass.getpass(" • Sudo password: ") + if not pw: + ui.warning("No password provided - assuming passwordless sudo") + return pw + + def _resolve_key_file(self) -> Path: + env_key = os.environ.get("FROMCHAT_SSH_KEY", "").strip() + if env_key: + return Path(env_key).expanduser() + ssh_dir = Path.home() / ".ssh" + for name in ("id_ed25519", "id_rsa"): + candidate = ssh_dir / name + if candidate.is_file(): + return candidate + return DEFAULT_KEY_FILE + + def _prompt_yes_no(self, message: str, *, default: bool = True) -> bool: + if not sys.stdin.isatty(): + return default + hint = "Y/n" if default else "y/N" + answer = input(f" {message} [{hint}]: ").strip() + if not answer: + return default + return answer.lower().startswith("y") + + def _ensure_agent(self) -> None: + if os.environ.get("SSH_AUTH_SOCK"): + return + subprocess.run(["ssh-agent", "-s"], capture_output=True, check=False) + + def _ensure_key_file(self, key_file: Path) -> None: + if key_file.is_file(): + return + ui.warning(f"SSH key not found at {key_file}") + print(f" Command: {KEYGEN_DISPLAY}") + if not self._prompt_yes_no("Create SSH key now?", default=True): + ui.error("SSH key is required for deploy.") + raise SystemExit(1) + key_file.parent.mkdir(mode=0o700, exist_ok=True) + ui.substep("Creating SSH key...") + result = subprocess.run( + [ + "ssh-keygen", + "-t", + "rsa", + "-b", + "4096", + "-f", + str(key_file), + "-N", + "", + "-C", + "fromchat-deploy", + ], + capture_output=True, + text=True, + ) + if result.returncode != 0: + ui.error("Failed to create SSH key") + if result.stderr: + print(result.stderr, file=sys.stderr) + raise SystemExit(1) + ui.success(f"Created {key_file}") + + def _ensure_key_in_agent(self, key_file: Path) -> None: + loaded = False + r = subprocess.run(["ssh-add", "-l"], capture_output=True, text=True) + if r.returncode == 0: + fp_r = subprocess.run( + ["ssh-keygen", "-lf", str(key_file)], + capture_output=True, + text=True, + ) + if fp_r.returncode == 0: + parts = fp_r.stdout.strip().split() + fingerprint = parts[1] if len(parts) > 1 else "" + if fingerprint and fingerprint in r.stdout: + loaded = True + if not loaded: + ui.substep("Adding SSH key to agent...") + add = subprocess.run(["ssh-add", str(key_file)], capture_output=True, text=True) + if add.returncode != 0: + ui.error("Failed to add SSH key to agent.") + if add.stderr: + print(add.stderr, file=sys.stderr) + raise SystemExit(1) + + def _server_host(self) -> str: + host = self._server.split("@", 1)[-1] + if host.startswith("[") and "]" in host: + return host[1 : host.index("]")] + return host.split(":", 1)[0] + + def _trust_host_key(self) -> None: + host = self._server_host() + if not host: + return + known_hosts = Path.home() / ".ssh" / "known_hosts" + known_hosts.parent.mkdir(mode=0o700, exist_ok=True) + if known_hosts.is_file(): + try: + with known_hosts.open(encoding="utf-8") as fh: + if host in fh.read(): + return + except OSError: + pass + ui.substep(f"Trusting host key for {host}...") + try: + scan = subprocess.run( + ["ssh-keyscan", "-H", host], + capture_output=True, + text=True, + timeout=30, + ) + except (subprocess.TimeoutExpired, OSError) as exc: + ui.warning(f"Could not scan host key for {host}: {exc}") + return + if scan.returncode != 0 or not scan.stdout.strip(): + ui.warning(f"ssh-keyscan returned no keys for {host}") + return + with known_hosts.open("a", encoding="utf-8") as fh: + fh.write(scan.stdout) + if not scan.stdout.endswith("\n"): + fh.write("\n") + ui.success(f"Host key for {host} added to known_hosts") + + def _test_key_auth(self) -> bool: + return ( + subprocess.run( + [ + "ssh", + "-o", + "BatchMode=yes", + *ssh_common_options(), + self._server, + "echo SSH key works", + ], + capture_output=True, + ).returncode + == 0 + ) + + def _install_pubkey(self, pub: Path, password: str) -> bool: + ui.substep(f"Installing public key on {self._server}...") + base_cmd = [ + "ssh-copy-id", + "-i", + str(pub), + *ssh_common_options(), + self._server, + ] + if password and shutil.which("sshpass"): + env = os.environ.copy() + env["SSHPASS"] = password + result = subprocess.run(["sshpass", "-e", *base_cmd], env=env) + return result.returncode == 0 + if password: + ui.substep("sshpass not found — enter the same password when ssh-copy-id prompts") + return subprocess.run(base_cmd).returncode == 0 + + def _verify_key_auth(self, key_file: Path, sudo_password: str) -> None: + pub = key_file.with_suffix(key_file.suffix + ".pub") + if not pub.is_file(): + ui.error(f"Missing public key: {pub}") + raise SystemExit(1) + + if self._test_key_auth(): + ui.success("SSH key authentication works") + return + + ui.warning(f"SSH key authentication failed for {self._server}") + if self._prompt_yes_no( + f"Install your public key on {self._server} with ssh-copy-id?", + default=True, + ): + if self._install_pubkey(pub, sudo_password) and self._test_key_auth(): + ui.success("SSH key installed and verified") + return + + ui.error("SSH key authentication still failing") + sys.stderr.write( + f' Try manually: ssh-copy-id -i "{pub}" "{self._server}"\n' + ) + if pub.is_file(): + sys.stderr.write(f" Public key: {pub.read_text(encoding='utf-8').strip()}\n") + raise SystemExit(1) + + def _verify_sudo_password(self, password: str) -> str: + if not password: + return "" + while True: + chk = subprocess.run( + ssh_argv(self._server, "sudo -S -v"), + input=(password + "\n").encode(), + capture_output=True, + ) + if chk.returncode == 0: + return password + ui.error("Invalid sudo password, please try again") + password = getpass.getpass(" • Sudo password: ") + if not password: + ui.warning("No password provided - assuming passwordless sudo") + return "" diff --git a/scripts/deploy/transfer.py b/scripts/deploy/transfer.py new file mode 100644 index 0000000..189230a --- /dev/null +++ b/scripts/deploy/transfer.py @@ -0,0 +1,289 @@ +"""Image pussh, rsync deployment layout, Firebase cert, remote systemd.""" + +from __future__ import annotations + +import shlex +import subprocess +import sys +from pathlib import Path + +from deploy.paths import ProjectPaths +from deploy.ssh_auth import SshCredentials, scp_argv, ssh_argv, ssh_common_options +import deploy.ui as ui + +UNREGISTRY_IMAGE = "ghcr.io/psviderski/unregistry" + +REMOTE_SYSTEMD_SCRIPT = r"""set -e + +REMOTE_SUDO_PASS="${SUDO_PASSWORD:-}" +REMOTE_DEPLOY_PATH="${DEPLOY_PATH:-}" +export SUDO_PROMPT="" + +sudo_cmd() { + if [ -n "$REMOTE_SUDO_PASS" ]; then + echo "$REMOTE_SUDO_PASS" | sudo -S -p '' "$@" 2>/dev/null + else + sudo "$@" 2>/dev/null + fi +} + +if [ -z "$REMOTE_DEPLOY_PATH" ]; then + echo "❌ DEPLOY_PATH is not set" + exit 1 +fi + +mkdir -p "$REMOTE_DEPLOY_PATH/config" "$REMOTE_DEPLOY_PATH/data/prod" +cd "$REMOTE_DEPLOY_PATH" + +if [ ! -f "$REMOTE_DEPLOY_PATH/.env" ]; then + echo "⚠️ Warning: .env file not found" +fi + +UNIT_SRC="$REMOTE_DEPLOY_PATH/fromchat.service" +if [ -f "$UNIT_SRC" ]; then + # Ensure WorkingDirectory matches deploy path + sed -i.bak "s|^WorkingDirectory=.*|WorkingDirectory=$REMOTE_DEPLOY_PATH|" "$UNIT_SRC" || true + sudo_cmd cp -f "$UNIT_SRC" /etc/systemd/system/fromchat.service +fi + +if systemctl is-active --quiet fromchat; then + sudo_cmd systemctl stop fromchat +fi + +docker compose down --remove-orphans > /dev/null 2>&1 || true + +sudo_cmd systemctl daemon-reload +sudo_cmd systemctl restart fromchat + +sleep 3 +if ! systemctl is-active --quiet fromchat; then + echo "❌ Service failed to start" + sudo_cmd journalctl --no-pager -xeu fromchat -n 30 + exit 1 +fi +""" + + +class DeployTransfer: + def __init__(self, paths: ProjectPaths) -> None: + self._paths = paths + + def ensure_pussh(self) -> None: + if subprocess.run(["docker", "pussh", "--help"], capture_output=True).returncode != 0: + ui.error("docker pussh plugin not installed") + print(" Install: npm run install:pussh (from the backend repo)") + + def ensure_unregistry(self, creds: SshCredentials) -> None: + """Server needs unregistry for pussh; pull locally then transfer if missing.""" + from deploy.util import image_exists_locally + + check = ( + "sudo docker images --format '{{.Repository}}:{{.Tag}}' | " + f"grep -q '^{UNREGISTRY_IMAGE}$'" + ) + if subprocess.run(ssh_argv(creds.server, check), capture_output=True).returncode == 0: + return + import deploy.docker_local as docker_local + + if not image_exists_locally(UNREGISTRY_IMAGE): + docker_local.pull_images([UNREGISTRY_IMAGE]) + ui.substep("Transferring unregistry image to server (one-time setup)…") + if subprocess.run(["docker", "pussh", UNREGISTRY_IMAGE, creds.server]).returncode != 0: + ui.error("Failed to transfer unregistry image to server") + raise SystemExit(1) + + def pussh_images(self, creds: SshCredentials, images: list[str]) -> None: + ui.step("Transferring images to server") + if not images: + ui.success("No images to transfer") + return + self.ensure_unregistry(creds) + for image in images: + ui.substep(f"Transferring {image}...") + if subprocess.run(["docker", "pussh", image, creds.server]).returncode != 0: + ui.error(f"Failed to transfer {image}") + raise SystemExit(1) + print() + + def pull_external_on_server(self, creds: SshCredentials, images: list[str]) -> None: + """Deprecated: classic deploy transfers all images from the local machine.""" + if images: + ui.warning( + "pull_external_on_server is unused; images are built/pulled locally and pussh'd." + ) + + def prepare_remote_dirs(self, creds: SshCredentials, deploy_path: str) -> None: + d_root = shlex.quote(deploy_path) + d_config = shlex.quote(f"{deploy_path}/config") + if creds.sudo_password: + pw = shlex.quote(creds.sudo_password) + script = f"""set -e +echo {pw} | sudo -S -p '' mkdir -p {d_root} {d_config} 2>/dev/null || true +echo {pw} | sudo -S -p '' chown -R $(whoami):$(whoami) {d_root} 2>/dev/null || true +""" + subprocess.run(ssh_argv(creds.server, "bash"), input=script.encode(), capture_output=True) + else: + subprocess.run( + ssh_argv( + creds.server, + f"sudo mkdir -p {d_root} {d_config} && " + f"sudo chown -R $(whoami):$(whoami) {d_root}", + ), + capture_output=True, + ) + + def rsync_staging(self, creds: SshCredentials, deploy_path: str, staging: Path) -> None: + ui.step("Transferring deployment files") + self.prepare_remote_dirs(creds, deploy_path) + ui.substep("Syncing install directory…") + rsync = subprocess.run( + [ + "rsync", + "-avz", + "--delete", + "-e", + "ssh " + " ".join(shlex.quote(o) for o in ssh_common_options()), + "--exclude", + ".env", + "--exclude", + "data/", + "--exclude", + "firebase-cert.json", + "--exclude", + "updater/", + f"{staging}/", + f"{creds.server}:{deploy_path}/", + ], + capture_output=True, + text=True, + ) + if rsync.returncode != 0: + ui.error("Rsync failed. Error output:") + for line in (rsync.stderr or rsync.stdout or "").splitlines(): + print(f" {line}") + raise SystemExit(1) + + def copy_env_prod(self, creds: SshCredentials, deploy_path: str) -> None: + candidates = [] + if self._paths.backend_dir: + candidates.append(self._paths.backend_dir / ".env.prod") + candidates.append(self._paths.deployment_root / ".env.prod") + prod = next((p for p in candidates if p.is_file()), None) + if prod: + ui.substep("Copying .env.prod to .env...") + if ( + subprocess.run( + scp_argv(str(prod), f"{creds.server}:{deploy_path}/.env"), + capture_output=True, + ).returncode + != 0 + ): + ui.warning("Failed to copy .env.prod to .env") + else: + ui.warning(".env.prod not found (looked in backend and deployment roots)") + + def resolve_deploy_path_on_server(self, server: str, deploy_path: str) -> str: + r = subprocess.run( + [ + "ssh", + "-o", + "BatchMode=yes", + *ssh_common_options(), + server, + f"eval echo {deploy_path}", + ], + capture_output=True, + text=True, + ) + out = r.stdout.strip() + return out if out else deploy_path + + def firebase_cert_path(self) -> Path: + if self._paths.backend_dir: + return self._paths.backend_dir / "firebase-cert.json" + return self._paths.deployment_root / "firebase-cert.json" + + def cleanup_remote_firebase_dir(self, creds: SshCredentials, deploy_path_resolved: str) -> None: + d = deploy_path_resolved + if creds.sudo_password: + pw = shlex.quote(creds.sudo_password) + script = f"""set -e +D={shlex.quote(d)} +C="$D/firebase-cert.json" +mkdir -p "$D" 2>/dev/null || true +if [ -d "$C" ]; then + echo {pw} | sudo -S -p '' rm -rf "$C" +fi +echo {pw} | sudo -S -p '' chown -R "$(whoami):$(whoami)" "$D" 2>/dev/null || true +""" + subprocess.run(ssh_argv(creds.server, "bash"), input=script.encode(), capture_output=True) + else: + q = shlex.quote(d) + subprocess.run( + ssh_argv( + creds.server, + f"D={q}; C=\"$D/firebase-cert.json\"; mkdir -p \"$D\"; " + f'if [ -d "$C" ]; then sudo rm -rf "$C" 2>/dev/null || rm -rf "$C"; fi; ' + f'sudo chown -R $(whoami):$(whoami) "$D" 2>/dev/null || true', + ), + capture_output=True, + ) + + def _wait_firebase_loop(self, cert: Path) -> None: + while True: + if cert.is_file(): + return + if cert.is_dir(): + print( + f" ⚠ {cert} is a directory. Delete it and save the Firebase " + "service account JSON as a file at that exact path." + ) + elif cert.exists(): + print(f" ⚠ {cert} exists but is not a regular file.") + else: + print(f" ⚠ Missing {cert} (Firebase service account JSON for FCM).") + print(" Fix this, then press Enter to check again (Ctrl+C to abort deploy).") + input() + + def sync_firebase_cert(self, creds: SshCredentials, deploy_path: str) -> str: + ui.substep("Firebase service account (runtime bind-mount: firebase-cert.json)...") + resolved = self.resolve_deploy_path_on_server(creds.server, deploy_path) + self.cleanup_remote_firebase_dir(creds, resolved) + cert = self.firebase_cert_path() + self._wait_firebase_loop(cert) + remote = f"{resolved}/firebase-cert.json" + self.scp_firebase(creds, cert, remote) + return resolved + + def scp_firebase(self, creds: SshCredentials, cert: Path, remote_path: str) -> None: + ui.substep("Copying firebase-cert.json...") + r = subprocess.run( + scp_argv(str(cert), f"{creds.server}:{remote_path}"), + capture_output=True, + text=True, + ) + if r.returncode != 0: + ui.error("Failed to copy firebase-cert.json to server") + print(f" Target: {creds.server}:{remote_path}", file=sys.stderr) + err = (r.stderr or r.stdout or "").strip() + if err: + for line in err.splitlines(): + print(f" {line}", file=sys.stderr) + raise SystemExit(1) + subprocess.run( + ssh_argv(creds.server, f"chmod 600 {shlex.quote(remote_path)}"), + capture_output=True, + ) + + def run_remote_systemd(self, creds: SshCredentials, deploy_path_resolved: str) -> None: + ui.step("Deploying on server") + pw = creds.sudo_password + dp = deploy_path_resolved + remote_cmd = f"SUDO_PASSWORD={shlex.quote(pw)} DEPLOY_PATH={shlex.quote(dp)} bash -s" + r = subprocess.run( + ssh_argv(creds.server, remote_cmd), + input=REMOTE_SYSTEMD_SCRIPT.encode(), + text=False, + ) + if r.returncode != 0: + raise SystemExit(r.returncode) diff --git a/scripts/deploy/ui.py b/scripts/deploy/ui.py new file mode 100644 index 0000000..15fdc2f --- /dev/null +++ b/scripts/deploy/ui.py @@ -0,0 +1,49 @@ +"""Terminal output (Rich), matching previous deploy.sh style.""" + +from __future__ import annotations + +from rich.console import Console +from rich.text import Text + +_console = Console(highlight=False) + +def banner() -> None: + _console.print() + _console.print(Text("🚀 Deployment", style="bold magenta")) + _console.print() + + +def build_banner() -> None: + _console.print() + _console.print(Text("🔨 Building Docker images", style="bold magenta")) + _console.print() + + +def deploy_banner(server: str) -> None: + _console.print() + _console.print(Text(f"🚀 Deploying to {server}", style="bold magenta")) + _console.print() + + +def info(msg: str) -> None: + _console.print(Text("ℹ ", style="blue"), msg, sep="") + + +def success(msg: str) -> None: + _console.print(Text("✓ ", style="green"), msg, sep="") + + +def warning(msg: str) -> None: + _console.print(Text("⚠ ", style="yellow"), msg, sep="") + + +def error(msg: str) -> None: + _console.print(Text("✗ ", style="red"), msg, sep="") + + +def step(msg: str) -> None: + _console.print(Text("→ ", style="bold cyan"), Text(msg, style="bold"), sep="") + + +def substep(msg: str, *, end: str = "\n") -> None: + _console.print(Text(" • ", style="green"), msg, sep="", end=end) diff --git a/scripts/deploy/updater.py b/scripts/deploy/updater.py new file mode 100644 index 0000000..6954b33 --- /dev/null +++ b/scripts/deploy/updater.py @@ -0,0 +1,169 @@ +"""Remote auto-updater install (matches install.sh setup_updater).""" + +from __future__ import annotations + +import getpass +import os +import shlex +import shutil +import subprocess +from pathlib import Path + +from deploy.paths import ProjectPaths +from deploy.ssh_auth import SshCredentials, ssh_argv, ssh_common_options +import deploy.ui as ui + +DEFAULT_BACKEND_REPO = "https://github.com/fromchat-messenger/backend.git" +DEFAULT_WEB_REPO = "https://github.com/fromchat-messenger/web.git" +DEFAULT_APP_REPO = "https://github.com/fromchat-messenger/app.git" +UPDATER_IMAGE = "fromchat/updater:latest" + + +def _repo_urls(paths: ProjectPaths) -> tuple[str, str, str]: + backend = os.environ.get("FROMCHAT_BACKEND_REPO", DEFAULT_BACKEND_REPO) + web = os.environ.get("FROMCHAT_WEB_REPO", DEFAULT_WEB_REPO) + app = os.environ.get("FROMCHAT_APP_REPO", DEFAULT_APP_REPO) + if paths.env_file.is_file(): + for line in paths.env_file.read_text(encoding="utf-8").splitlines(): + line = line.strip() + if not line or line.startswith("#") or "=" not in line: + continue + key, _, val = line.partition("=") + key = key.strip() + val = val.strip().strip('"').strip("'") + if key == "FROMCHAT_BACKEND_REPO" and val: + backend = val + elif key == "FROMCHAT_WEB_REPO" and val: + web = val + elif key == "FROMCHAT_APP_REPO" and val: + app = val + return backend, web, app + + +def resolve_git_token(explicit: str | None) -> str: + if explicit: + return explicit + for key in ("GIT_TOKEN", "GITHUB_TOKEN", "RELEASES_TOKEN"): + val = os.environ.get(key, "").strip() + if val: + return val + ui.step("Git token for the auto-updater (GitHub or Gitea)") + print(" GitHub: https://github.com/settings/tokens/new?scopes=read:packages,repo") + print(" Gitea: Settings → Applications → Generate New Token") + token = getpass.getpass(" Paste token (input hidden): ").strip() + if not token: + ui.error("Git token is required when updater is selected.") + raise SystemExit(1) + return token + + +def _updater_compose_source(paths: ProjectPaths) -> Path: + parent = paths.deployment_root.parent + candidates = [ + paths.deployment_root.parent / "updater" / "compose.yml", + parent / "updater" / "compose.yml", + ] + env_dir = os.environ.get("FROMCHAT_UPDATER_DIR", "") + if env_dir: + candidates.insert(0, Path(env_dir).expanduser() / "compose.yml") + for cand in candidates: + if cand.is_file(): + return cand + ui.error( + "updater/compose.yml not found. Set FROMCHAT_UPDATER_DIR or keep ../updater sibling." + ) + raise SystemExit(1) + + +def write_updater_env( + dest: Path, + *, + token: str, + deploy_path_resolved: str, + components: list[str], + paths: ProjectPaths, +) -> None: + backend, web, app = _repo_urls(paths) + components_csv = ",".join(components) + dest.write_text( + "\n".join( + [ + f"GITHUB_TOKEN={token}", + f"GIT_TOKEN={token}", + f"BACKEND_REPO={backend}", + f"WEB_REPO={web}", + f"DEPLOYMENT_REPO={app}", + f"COMPOSE_PROJECT_DIR={deploy_path_resolved}", + f"FROMCHAT_COMPONENTS={components_csv}", + "CHECK_INTERVAL_SECONDS=60", + "", + ] + ), + encoding="utf-8", + ) + + +def setup_updater_remote( + creds: SshCredentials, + deploy_path: str, + deploy_path_resolved: str, + *, + components: list[str], + paths: ProjectPaths, + git_token: str | None, + sudo_password: str, +) -> None: + ui.step("Setting up auto-updater on server") + token = resolve_git_token(git_token) + + staging = paths.staging_dir / "updater" + if staging.exists(): + shutil.rmtree(staging) + staging.mkdir(parents=True) + + compose_src = _updater_compose_source(paths) + shutil.copy2(compose_src, staging / "compose.yml") + write_updater_env( + staging / ".env", + token=token, + deploy_path_resolved=deploy_path_resolved, + components=components, + paths=paths, + ) + + remote_updater = f"{deploy_path}/updater" + ui.substep("Syncing updater/ to server…") + subprocess.run( + ssh_argv(creds.server, f"mkdir -p {shlex.quote(remote_updater)}"), + check=True, + capture_output=True, + ) + rsync = subprocess.run( + [ + "rsync", + "-avz", + "-e", + "ssh " + " ".join(shlex.quote(o) for o in ssh_common_options()), + f"{staging}/", + f"{creds.server}:{remote_updater}/", + ], + capture_output=True, + text=True, + ) + if rsync.returncode != 0: + ui.error("Failed to sync updater directory") + for line in (rsync.stderr or rsync.stdout or "").splitlines(): + print(f" {line}") + raise SystemExit(1) + + ui.substep("Starting updater service…") + remote_cmd = ( + f"cd {shlex.quote(remote_updater)} && " + f"COMPOSE_PROJECT_DIR={shlex.quote(deploy_path_resolved)} " + f"docker compose --env-file .env up -d --wait --timeout 120" + ) + if subprocess.run(ssh_argv(creds.server, remote_cmd), capture_output=True).returncode != 0: + ui.error("Failed to start updater on server") + raise SystemExit(1) + + ui.success("Updater service started on server") diff --git a/scripts/deploy/util.py b/scripts/deploy/util.py new file mode 100644 index 0000000..61982ef --- /dev/null +++ b/scripts/deploy/util.py @@ -0,0 +1,99 @@ +"""Small helpers: hashing, dedupe, cache keys.""" + +from __future__ import annotations + +import hashlib +import subprocess +import sys +from pathlib import Path + + +def sanitize_ref(ref: str) -> str: + s = ref.replace("/", "_").replace(":", "__").replace("@", "__at__") + return s + + +def dedupe_preserve(items: list[str]) -> list[str]: + seen: set[str] = set() + out: list[str] = [] + for x in items: + if x not in seen: + seen.add(x) + out.append(x) + return out + + +def read_file_if_exists(path: Path) -> str: + if path.is_file(): + return path.read_text(encoding="utf-8", errors="replace") + return "" + + +def local_image_layer_fp(image: str) -> str: + def inspect_layers(ref: str) -> subprocess.CompletedProcess[str]: + return subprocess.run( + ["docker", "image", "inspect", "-f", "{{json .RootFS.Layers}}", ref], + capture_output=True, + text=True, + ) + + p = inspect_layers(image) + if p.returncode != 0: + # Docker Desktop occasionally ends up in a state where repo:tag exists in `docker images` + # but `docker image inspect repo:tag` fails. Inspecting by content-addressed ID works. + id_p = subprocess.run( + ["docker", "images", "--no-trunc", "--format", "{{.ID}}", image], + capture_output=True, + text=True, + ) + image_id = (id_p.stdout or "").strip() + if not image_id: + return "" + p = inspect_layers(image_id) + if p.returncode != 0: + return "" + + return hashlib.sha256(p.stdout.encode()).hexdigest() + + +def compute_inputs_hash( + context: Path, + dockerfile: Path, + *, + hash_script: Path, + python_exe: str | None = None, +) -> str: + exe = python_exe or sys.executable + p = subprocess.run( + [exe, str(hash_script), "--context", str(context), "--dockerfile", str(dockerfile)], + capture_output=True, + text=True, + ) + if p.returncode != 0: + return "" + return p.stdout.strip() + + +def local_docker_image_tags() -> set[str]: + p = subprocess.run( + ["docker", "images", "--format", "{{.Repository}}:{{.Tag}}"], + capture_output=True, + text=True, + ) + if p.returncode != 0: + return set() + return {line.strip() for line in p.stdout.splitlines() if line.strip()} + + +def image_exists_locally(image: str) -> bool: + return image in local_docker_image_tags() + + +def require_local_images(images: list[str], *, label: str) -> None: + missing = [img for img in dedupe_preserve(images) if not image_exists_locally(img)] + if missing: + raise RuntimeError( + f"{label} not found locally: {', '.join(missing)}\n" + "Build or pull them on this machine first, then deploy offline." + ) + diff --git a/scripts/docker_inputs_hash.py b/scripts/docker_inputs_hash.py new file mode 100644 index 0000000..c1add15 --- /dev/null +++ b/scripts/docker_inputs_hash.py @@ -0,0 +1,369 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import fnmatch +import hashlib +import json +import os +import shlex +import sys +from dataclasses import dataclass +from pathlib import Path + + +def _sha256_bytes(data: bytes) -> str: + h = hashlib.sha256() + h.update(data) + return h.hexdigest() + + +def _sha256_file(path: Path) -> str: + h = hashlib.sha256() + with path.open("rb") as f: + for chunk in iter(lambda: f.read(1024 * 1024), b""): + h.update(chunk) + return h.hexdigest() + + +def _read_text(path: Path) -> str: + return path.read_text(encoding="utf-8", errors="replace") + + +@dataclass(frozen=True) +class DockerIgnoreRule: + pattern: str + negated: bool + anchored: bool + directory_only: bool + + +def _read_dockerignore_rules(context: Path) -> list[DockerIgnoreRule]: + p = context / ".dockerignore" + if not p.exists() or not p.is_file(): + return [] + + rules: list[DockerIgnoreRule] = [] + for raw in _read_text(p).splitlines(): + line = raw.strip() + if not line or line.startswith("#"): + continue + negated = line.startswith("!") + if negated: + line = line[1:].lstrip() + anchored = line.startswith("/") + if anchored: + line = line[1:] + directory_only = line.endswith("/") + if directory_only: + line = line[:-1] + if not line: + continue + rules.append( + DockerIgnoreRule( + pattern=line, + negated=negated, + anchored=anchored, + directory_only=directory_only, + ) + ) + return rules + + +def _dockerignore_matches(rule: DockerIgnoreRule, rel_posix: str, is_dir: bool) -> bool: + if rule.directory_only and not is_dir: + return False + + rel = rel_posix.lstrip("./") + if rule.anchored: + # Anchored to context root. + candidates = [rel] + else: + # Unanchored patterns match anywhere: try both full rel and basename. + base = rel.rsplit("/", 1)[-1] + candidates = [rel, base] + + # Dockerignore supports ** globs; fnmatch handles this well enough for our use. + for c in candidates: + if fnmatch.fnmatch(c, rule.pattern): + return True + # Also allow matching directory prefixes for patterns like "dist" against "foo/dist/bar". + if not rule.anchored and "/" in rel: + if fnmatch.fnmatch(rel, f"*/{rule.pattern}") or fnmatch.fnmatch(rel, f"**/{rule.pattern}"): + return True + return False + + +def _is_ignored_by_dockerignore(rules: list[DockerIgnoreRule], rel_posix: str, is_dir: bool) -> bool: + ignored = False + for r in rules: + if _dockerignore_matches(r, rel_posix=rel_posix, is_dir=is_dir): + ignored = not r.negated + return ignored + + +def _dockerfile_logical_lines(dockerfile_text: str) -> list[str]: + """ + Join backslash-continued lines and drop full-line comments. + """ + out: list[str] = [] + buf: list[str] = [] + for raw in dockerfile_text.splitlines(): + line = raw.rstrip() + if not buf: + stripped = line.lstrip() + if stripped.startswith("#") or stripped == "": + continue + buf.append(line) + if line.endswith("\\"): + buf[-1] = buf[-1][:-1].rstrip() + continue + joined = " ".join(x.strip() for x in buf if x.strip()) + buf = [] + if joined: + out.append(joined) + if buf: + joined = " ".join(x.strip() for x in buf if x.strip()) + if joined: + out.append(joined) + return out + + +@dataclass(frozen=True) +class CopyAdd: + sources: tuple[str, ...] + from_stage: bool + + +def _parse_copy_add_args_shellform(args: list[str]) -> CopyAdd | None: + # flags: --from=, --chown=, --chmod=, --link, --parents, --exclude=... etc. + from_stage = False + rest: list[str] = [] + for a in args: + if a.startswith("--from=") or a == "--from": + from_stage = True + continue + if a.startswith("--"): + continue + rest.append(a) + if len(rest) < 2: + return None + # last is dest + srcs = tuple(rest[:-1]) + return CopyAdd(sources=srcs, from_stage=from_stage) + + +def _parse_copy_add_args_jsonform(json_text: str) -> CopyAdd | None: + try: + arr = json.loads(json_text) + except Exception: + return None + if not isinstance(arr, list) or len(arr) < 2: + return None + # last is dest + srcs = tuple(x for x in arr[:-1] if isinstance(x, str)) + if not srcs: + return None + return CopyAdd(sources=srcs, from_stage=False) + + +def _parse_copy_add(line: str) -> CopyAdd | None: + upper = line.lstrip().upper() + if not (upper.startswith("COPY ") or upper.startswith("ADD ")): + return None + + # Keep original casing for paths. + keyword, rest = line.split(None, 1) + rest = rest.strip() + + # JSON form starts with '[' + if rest.startswith("["): + parsed = _parse_copy_add_args_jsonform(rest) + if parsed: + return parsed + return None + + # shell form + try: + parts = shlex.split(rest, posix=True) + except Exception: + return None + return _parse_copy_add_args_shellform(parts) + + +def _looks_like_remote(src: str) -> bool: + s = src.lower() + return s.startswith("http://") or s.startswith("https://") + + +def _is_glob(p: str) -> bool: + return any(ch in p for ch in ["*", "?", "["]) + + +def _iter_files_under(path: Path) -> list[Path]: + if not path.exists(): + return [] + if path.is_file(): + return [path] + files: list[Path] = [] + for root, _, filenames in os.walk(path): + for name in filenames: + files.append(Path(root) / name) + return files + + +def _collect_sources(context: Path, dockerfile_path: Path) -> list[Path]: + text = _read_text(dockerfile_path) + logical = _dockerfile_logical_lines(text) + dockerignore_rules = _read_dockerignore_rules(context) + + paths: list[Path] = [] + for ln in logical: + parsed = _parse_copy_add(ln) + if not parsed: + continue + if parsed.from_stage: + continue + for src in parsed.sources: + if _looks_like_remote(src): + continue + if src.startswith("/"): + # Absolute COPY sources aren't valid for local context; ignore to avoid surprises. + continue + # Docker allows ".", "./foo", etc. + src_norm = src.lstrip("./") + if src_norm == "": + src_norm = "." + + if _is_glob(src_norm): + # Expand within context + for root, _, filenames in os.walk(context): + root_p = Path(root) + rel_root = root_p.relative_to(context).as_posix() + for fn in filenames: + rel = f"{rel_root}/{fn}" if rel_root != "." else fn + if _is_ignored_by_dockerignore(dockerignore_rules, rel_posix=rel, is_dir=False): + continue + if fnmatch.fnmatch(rel, src_norm) or fnmatch.fnmatch(fn, src_norm): + paths.append(context / rel) + continue + + p = (context / src_norm).resolve() + # Ensure stays within context + try: + p.relative_to(context.resolve()) + except Exception: + continue + for fp in _iter_files_under(p): + try: + rel = fp.resolve().relative_to(context.resolve()).as_posix() + except Exception: + continue + if _is_ignored_by_dockerignore(dockerignore_rules, rel_posix=rel, is_dir=fp.is_dir()): + continue + paths.append(fp) + + # Always include the Dockerfile itself (and preserve stable ordering via sort later) + paths.append(dockerfile_path.resolve()) + return paths + + +def compute_inputs_hash(context: Path, dockerfile_path: Path) -> str: + files = _collect_sources(context=context, dockerfile_path=dockerfile_path) + # Deduplicate by resolved path + uniq: dict[str, Path] = {} + for p in files: + uniq[str(p)] = p + + # Stable sort by path relative to context when possible, else absolute + ctx_resolved = context.resolve() + def sort_key(p: Path) -> str: + try: + return p.resolve().relative_to(ctx_resolved).as_posix() + except Exception: + return p.resolve().as_posix() + + sorted_files = sorted(uniq.values(), key=sort_key) + + h = hashlib.sha256() + for p in sorted_files: + rp: str + try: + rp = p.resolve().relative_to(ctx_resolved).as_posix() + except Exception: + rp = p.resolve().as_posix() + h.update(rp.encode("utf-8", errors="strict")) + h.update(b"\0") + if p.is_file(): + h.update(_sha256_file(p).encode("ascii")) + else: + h.update(b"NONFILE") + h.update(b"\n") + + return h.hexdigest() + + +def compute_inputs_debug(context: Path, dockerfile_path: Path) -> tuple[str, list[tuple[str, str]]]: + """ + Returns (inputs_hash, [(rel_path, sha256_of_file_contents), ...]) with dockerignore applied. + """ + files = _collect_sources(context=context, dockerfile_path=dockerfile_path) + uniq: dict[str, Path] = {} + for p in files: + uniq[str(p.resolve())] = p.resolve() + + ctx_resolved = context.resolve() + + def rel_or_abs(p: Path) -> str: + try: + return p.resolve().relative_to(ctx_resolved).as_posix() + except Exception: + return p.resolve().as_posix() + + sorted_files = sorted(uniq.values(), key=lambda p: rel_or_abs(p)) + + items: list[tuple[str, str]] = [] + for p in sorted_files: + rp = rel_or_abs(p) + if p.is_file(): + items.append((rp, _sha256_file(p))) + else: + items.append((rp, "NONFILE")) + + return compute_inputs_hash(context=context, dockerfile_path=dockerfile_path), items + + +def main() -> int: + ap = argparse.ArgumentParser() + ap.add_argument("--context", required=True, help="Build context directory") + ap.add_argument("--dockerfile", required=True, help="Dockerfile path") + ap.add_argument( + "--debug-list", + action="store_true", + help="Print the included file list (relpath|sha256) to stderr", + ) + args = ap.parse_args() + + context = Path(args.context).resolve() + dockerfile = Path(args.dockerfile).resolve() + + if not context.exists() or not context.is_dir(): + print(f"Context not found or not a directory: {context}", file=sys.stderr) + return 2 + if not dockerfile.exists() or not dockerfile.is_file(): + print(f"Dockerfile not found: {dockerfile}", file=sys.stderr) + return 2 + + if args.debug_list: + h, items = compute_inputs_debug(context=context, dockerfile_path=dockerfile) + for rp, sh in items: + print(f"{rp}|{sh}", file=sys.stderr) + print(h) + else: + print(compute_inputs_hash(context=context, dockerfile_path=dockerfile)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) + diff --git a/scripts/generate-compose.py b/scripts/generate-compose.py new file mode 100644 index 0000000..86d1f60 --- /dev/null +++ b/scripts/generate-compose.py @@ -0,0 +1,197 @@ +#!/usr/bin/env python3 +""" +Merge backend + frontend (+ optional caddy) compose files for production deploy. +Replaces build: stanzas with image: fromchat/:. +""" +from __future__ import annotations + +import argparse +import copy +import sys +from pathlib import Path +from typing import Any + +import yaml + + +# Services that map to fromchat/* images when they use build: +FROMCHAT_IMAGE_SERVICES = frozenset( + {"main", "messaging", "file_storage", "postgres", "web", "caddy"} +) + +BACKEND_SERVICES = frozenset( + {"main", "messaging", "file_storage", "livekit", "postgres"} +) +FRONTEND_SERVICES = frozenset({"web"}) +CADDY_SERVICES = frozenset({"caddy"}) + +STRIP_KEYS = ("develop",) + + +def load_yaml(path: Path) -> dict[str, Any]: + with path.open(encoding="utf-8") as fh: + data = yaml.safe_load(fh) or {} + if not isinstance(data, dict): + raise ValueError(f"{path}: root must be a mapping") + return data + + +def dump_yaml(data: dict[str, Any], path: Path) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("w", encoding="utf-8") as fh: + yaml.dump( + data, + fh, + default_flow_style=False, + sort_keys=False, + allow_unicode=True, + ) + + +def deep_merge(base: dict[str, Any], extra: dict[str, Any]) -> dict[str, Any]: + out = copy.deepcopy(base) + for key, value in extra.items(): + if key in out and isinstance(out[key], dict) and isinstance(value, dict): + out[key] = deep_merge(out[key], value) + else: + out[key] = copy.deepcopy(value) + return out + + +def strip_dev_keys(service: dict[str, Any]) -> None: + for key in STRIP_KEYS: + service.pop(key, None) + + +def replace_build_with_image( + service_name: str, + service: dict[str, Any], + tag: str, +) -> None: + if "build" not in service: + return + if service_name not in FROMCHAT_IMAGE_SERVICES: + del service["build"] + return + del service["build"] + service["image"] = f"fromchat/{service_name}:{tag}" + + +def patch_livekit_volume(service: dict[str, Any]) -> None: + volumes = service.get("volumes") + if not volumes: + return + patched: list[Any] = [] + for entry in volumes: + if isinstance(entry, str) and "src/livekit/compose.yaml" in entry: + patched.append("./config/livekit.yaml:/etc/livekit.yaml:ro") + else: + patched.append(entry) + service["volumes"] = patched + + +def filter_services( + services: dict[str, Any], + enabled: set[str], +) -> dict[str, Any]: + return {name: cfg for name, cfg in services.items() if name in enabled} + + +def generate( + *, + backend_compose: Path | None, + frontend_compose: Path | None, + caddy_compose: Path | None, + tag: str, + components: set[str], + output: Path, +) -> None: + merged: dict[str, Any] = {"services": {}, "networks": {}, "volumes": {}} + + enabled: set[str] = set() + if "backend" in components: + enabled |= BACKEND_SERVICES + if "frontend" in components: + enabled |= FRONTEND_SERVICES + if "caddy" in components: + enabled |= CADDY_SERVICES + + sources: list[Path] = [] + if backend_compose and backend_compose.is_file(): + sources.append(backend_compose) + if frontend_compose and frontend_compose.is_file(): + sources.append(frontend_compose) + if caddy_compose and caddy_compose.is_file() and "caddy" in components: + sources.append(caddy_compose) + + if not sources: + raise SystemExit("No compose inputs found.") + + for src in sources: + doc = load_yaml(src) + merged = deep_merge(merged, doc) + + services: dict[str, Any] = merged.get("services") or {} + services = filter_services(services, enabled) + + for name, cfg in services.items(): + strip_dev_keys(cfg) + replace_build_with_image(name, cfg, tag) + if name == "livekit": + patch_livekit_volume(cfg) + if name == "caddy": + cfg.setdefault("volumes", []) + vols = cfg["volumes"] + if isinstance(vols, list) and not any("Caddyfile" in str(v) for v in vols): + vols.append("./Caddyfile:/etc/caddy/Caddyfile:ro") + cfg.setdefault("networks", ["public"]) + cfg.setdefault("depends_on", ["web"]) + cfg.setdefault("restart", "unless-stopped") + cfg.setdefault("ports", ["80:80", "443:443"]) + + if "caddy" in components and "web" in services: + web = services["web"] + ports = web.get("ports") + if isinstance(ports, list): + web["ports"] = [p for p in ports if not str(p).startswith("8301:")] + + merged["services"] = services + if not merged.get("networks"): + merged.pop("networks", None) + if not merged.get("volumes"): + merged.pop("volumes", None) + + dump_yaml(merged, output) + print(f"Wrote {output} ({len(services)} services, tag {tag})", file=sys.stderr) + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Generate FromChat production compose.yml") + parser.add_argument("--tag", required=True, help="Image tag, e.g. v1.0.0") + parser.add_argument( + "--components", + required=True, + help="Comma-separated: backend,frontend,caddy", + ) + parser.add_argument("--backend-compose", type=Path, default=None) + parser.add_argument("--frontend-compose", type=Path, default=None) + parser.add_argument("--caddy-compose", type=Path, default=None) + parser.add_argument("--output", type=Path, default=Path("compose.yml")) + return parser.parse_args() + + +def main() -> None: + args = parse_args() + components = {c.strip().lower() for c in args.components.split(",") if c.strip()} + generate( + backend_compose=args.backend_compose, + frontend_compose=args.frontend_compose, + caddy_compose=args.caddy_compose, + tag=args.tag, + components=components, + output=args.output, + ) + + +if __name__ == "__main__": + main() diff --git a/scripts/git_remote.py b/scripts/git_remote.py new file mode 100644 index 0000000..26cd0c9 --- /dev/null +++ b/scripts/git_remote.py @@ -0,0 +1,378 @@ +#!/usr/bin/env python3 +""" +GitHub + Gitea (git.fromchat.ru) helpers for FromChat deployment/updater. + +Official GitHub: github.com/fromchat-messenger/{backend,web,app} +Gitea: git.fromchat.ru/FromChat/{same repo name} + +Official GitHub URLs fall back to Gitea when GitHub fails. +URLs already on git.fromchat.ru use Gitea APIs directly. +""" +from __future__ import annotations + +import json +import re +import sys +import urllib.error +import urllib.parse +import urllib.request + +SEMVER = re.compile(r"^v(\d+)\.(\d+)(?:\.(\d+))?$") + +GITHUB_ORG = "fromchat-messenger" +GITEA_ORG = "FromChat" +GITHUB_OFFICIAL_PREFIX = f"https://github.com/{GITHUB_ORG}/" +GITEA_BASE = "https://git.fromchat.ru" + +OFFICIAL_GITHUB_REPOS = frozenset( + { + f"https://github.com/{GITHUB_ORG}/backend.git", + f"https://github.com/{GITHUB_ORG}/web.git", + f"https://github.com/{GITHUB_ORG}/app.git", + } +) + + +def normalize_repo_url(url: str) -> str: + u = url.rstrip("/") + if u.endswith(".git"): + return u + return u + ".git" + + +def is_gitea_repo(url: str) -> bool: + return "git.fromchat.ru" in normalize_repo_url(url) + + +def is_official_github_repo(url: str) -> bool: + normalized = normalize_repo_url(url) + return normalized in OFFICIAL_GITHUB_REPOS or GITHUB_OFFICIAL_PREFIX in normalized + + +def gitea_owner_repo(github_owner: str, repo: str) -> tuple[str, str]: + """Map github.com/fromchat-messenger/{repo} -> git.fromchat.ru/FromChat/{repo}.""" + if github_owner == GITEA_ORG: + return github_owner, repo + if github_owner == GITHUB_ORG or is_official_github_repo( + f"https://github.com/{github_owner}/{repo}.git" + ): + return GITEA_ORG, repo + return github_owner, repo + + +def parse_slug(url: str) -> tuple[str, str]: + u = url.rstrip("/").removesuffix(".git") + for marker in ("github.com/", "git.fromchat.ru/"): + if marker in u: + slug = u.split(marker, 1)[1] + owner, repo = slug.split("/", 1) + return owner, repo + if "/" in u and "://" not in u: + owner, repo = u.split("/", 1) + return owner, repo + raise ValueError(f"Cannot parse repository slug from {url!r}") + + +def _auth_headers(token: str | None, *, gitea: bool = False) -> dict[str, str]: + headers = {"User-Agent": "fromchat-git-remote"} + if not token: + return headers + if gitea: + # Gitea accepts both schemes; token is the documented form. + headers["Authorization"] = f"token {token}" + else: + headers["Authorization"] = f"Bearer {token}" + return headers + + +def _http_get(url: str, token: str | None, *, gitea: bool = False) -> bytes: + req = urllib.request.Request( + url, + headers={ + **_auth_headers(token, gitea=gitea), + **({"Accept": "application/vnd.github+json"} if not gitea else {}), + }, + ) + with urllib.request.urlopen(req, timeout=45) as resp: + return resp.read() + + +def github_raw_url(owner: str, repo: str, ref: str, path: str) -> str: + return f"https://raw.githubusercontent.com/{owner}/{repo}/{ref}/{path.lstrip('/')}" + + +def gitea_api_raw_url(gitea_owner: str, gitea_repo: str, ref: str, path: str) -> str: + """Gitea API raw file (supports branch/tag/commit via ?ref=).""" + filepath = urllib.parse.quote(path.lstrip("/"), safe="/") + ref_q = urllib.parse.quote(ref, safe="") + return ( + f"{GITEA_BASE}/api/v1/repos/{gitea_owner}/{gitea_repo}/raw/{filepath}?ref={ref_q}" + ) + + +def gitea_web_raw_urls(gitea_owner: str, gitea_repo: str, ref: str, path: str) -> list[str]: + """Legacy/browser raw paths (tried if API raw fails).""" + p = path.lstrip("/") + ref_q = urllib.parse.quote(ref, safe="") + return [ + f"{GITEA_BASE}/{gitea_owner}/{gitea_repo}/raw/{ref_q}/{p}", + f"{GITEA_BASE}/{gitea_owner}/{gitea_repo}/raw/tag/{ref_q}/{p}", + ] + + +def fetch_gitea_raw( + gitea_owner: str, + gitea_repo: str, + ref: str, + path: str, + token: str | None, +) -> bytes: + urls = [gitea_api_raw_url(gitea_owner, gitea_repo, ref, path)] + urls.extend(gitea_web_raw_urls(gitea_owner, gitea_repo, ref, path)) + + last_error: Exception | None = None + for url in urls: + try: + return _http_get(url, token, gitea=True) + except (urllib.error.HTTPError, urllib.error.URLError, TimeoutError) as exc: + last_error = exc + raise RuntimeError( + f"Failed to fetch {path} from git.fromchat.ru/{gitea_owner}/{gitea_repo}@{ref}" + ) from last_error + + +def fetch_raw_file(repo_url: str, ref: str, path: str, token: str | None = None) -> bytes: + owner, repo = parse_slug(repo_url) + + if is_gitea_repo(repo_url): + gitea_owner, gitea_repo = gitea_owner_repo(owner, repo) + return fetch_gitea_raw(gitea_owner, gitea_repo, ref, path, token) + + try: + return _http_get(github_raw_url(owner, repo, ref, path), token, gitea=False) + except (urllib.error.HTTPError, urllib.error.URLError, TimeoutError) as github_err: + if not is_official_github_repo(repo_url): + raise RuntimeError( + f"Failed to fetch {path} from {repo_url}@{ref}" + ) from github_err + gitea_owner, gitea_repo = gitea_owner_repo(owner, repo) + print( + f"GitHub unavailable for {owner}/{repo}, " + f"trying git.fromchat.ru/{gitea_owner}/{gitea_repo}…", + file=sys.stderr, + ) + return fetch_gitea_raw(gitea_owner, gitea_repo, ref, path, token) + + +def _tags_from_github(owner: str, repo: str, token: str | None) -> set[str]: + data = json.loads( + _http_get( + f"https://api.github.com/repos/{owner}/{repo}/tags?per_page=100", + token, + gitea=False, + ).decode() + ) + return {item["name"] for item in data if SEMVER.fullmatch(item.get("name", ""))} + + +def _gitea_paginated_json(path: str, token: str | None) -> list[dict]: + items: list[dict] = [] + page = 1 + while True: + sep = "&" if "?" in path else "?" + url = f"{GITEA_BASE}{path}{sep}page={page}&limit=50" + chunk = json.loads(_http_get(url, token, gitea=True).decode()) + if not isinstance(chunk, list): + break + if not chunk: + break + items.extend(chunk) + if len(chunk) < 50: + break + page += 1 + return items + + +def _tags_from_gitea(gitea_owner: str, gitea_repo: str, token: str | None) -> set[str]: + out: set[str] = set() + + for item in _gitea_paginated_json( + f"/api/v1/repos/{gitea_owner}/{gitea_repo}/tags", + token, + ): + name = item.get("name", "") + if SEMVER.fullmatch(name): + out.add(name) + + if out: + return out + + # Fallback: release tags + for item in _gitea_paginated_json( + f"/api/v1/repos/{gitea_owner}/{gitea_repo}/releases", + token, + ): + name = item.get("tag_name") or item.get("tag") or "" + if SEMVER.fullmatch(name): + out.add(name) + + return out + + +def fetch_semver_tags(repo_url: str, token: str | None = None) -> set[str]: + owner, repo = parse_slug(repo_url) + + if is_gitea_repo(repo_url): + gitea_owner, gitea_repo = gitea_owner_repo(owner, repo) + return _tags_from_gitea(gitea_owner, gitea_repo, token) + + try: + return _tags_from_github(owner, repo, token) + except (urllib.error.HTTPError, urllib.error.URLError, TimeoutError, json.JSONDecodeError): + if not is_official_github_repo(repo_url): + raise + gitea_owner, gitea_repo = gitea_owner_repo(owner, repo) + print( + f"GitHub tags unavailable for {owner}/{repo}, " + f"trying git.fromchat.ru/{gitea_owner}/{gitea_repo}…", + file=sys.stderr, + ) + return _tags_from_gitea(gitea_owner, gitea_repo, token) + + +def semver_key(tag: str) -> tuple[int, ...]: + m = SEMVER.match(tag) + if not m: + raise ValueError(tag) + return (int(m.group(1)), int(m.group(2)), int(m.group(3) or 0)) + + +def resolve_common_tag(repo_urls: list[str], token: str | None = None) -> str: + sets = [fetch_semver_tags(url, token) for url in repo_urls] + common = set.intersection(*sets) if sets else set() + if not common: + raise RuntimeError("No common semver tag (vX.Y or vX.Y.Z) found across repos.") + return max(common, key=semver_key) + + +def _image_tag_matches(needle: str, tag: str, candidates: list[str]) -> bool: + n = needle.lstrip("v") + t = tag.lstrip("v") + for c in candidates: + if not c: + continue + c = str(c) + if c in (tag, needle, t, n, f"v{n}"): + return True + return False + + +def package_exists_github(github_owner: str, package: str, tag: str, token: str) -> bool: + short = package.split("/", 1)[-1] + url = ( + f"https://api.github.com/users/{github_owner}/packages/container/{short}/versions" + "?per_page=20" + ) + try: + data = json.loads(_http_get(url, token, gitea=False).decode()) + except (urllib.error.HTTPError, urllib.error.URLError): + return False + if not isinstance(data, list): + return False + for version in data: + meta = version.get("metadata", {}).get("container", {}) + tags = meta.get("tags") or [] + names = version.get("name") or version.get("version") or "" + if _image_tag_matches(tag, tag, list(tags) + [names]): + return True + return False + + +def package_exists_gitea(gitea_owner: str, package: str, tag: str, token: str) -> bool: + short = package.split("/", 1)[-1] + path = f"/api/v1/packages/{gitea_owner}/container/{short}/versions" + try: + versions = _gitea_paginated_json(path, token) + except (urllib.error.HTTPError, urllib.error.URLError, json.JSONDecodeError): + return False + + for version in versions: + names: list[str] = [] + for key in ("version", "name"): + val = version.get(key) + if isinstance(val, str): + names.append(val) + metadata = version.get("metadata") or {} + if isinstance(metadata, dict): + container = metadata.get("container") or {} + if isinstance(container, dict): + names.extend(str(t) for t in (container.get("tags") or [])) + tags_field = metadata.get("tags") + if isinstance(tags_field, list): + names.extend(str(t) for t in tags_field) + if _image_tag_matches(tag, tag, names): + return True + return False + + +def package_version_exists( + github_owner: str, + package: str, + tag: str, + token: str, + *, + official: bool = True, +) -> bool: + gitea_owner, _ = gitea_owner_repo(github_owner, package.split("/", 1)[-1]) + + if github_owner == GITEA_ORG: + return package_exists_gitea(gitea_owner, package, tag, token) + + if package_exists_github(github_owner, package, tag, token): + return True + if not official: + return False + if package_exists_gitea(gitea_owner, package, tag, token): + print( + f"GitHub packages unavailable for {package}, confirmed on git.fromchat.ru", + file=sys.stderr, + ) + return True + return False + + +def resolve_git_token() -> str | None: + import os + + return os.environ.get("GIT_TOKEN") or os.environ.get("GITHUB_TOKEN") or None + + +def main() -> None: + if len(sys.argv) < 2: + sys.exit( + "usage: git_remote.py fetch-raw REPO REF PATH [TOKEN] | " + "resolve-tag [TOKEN] REPO..." + ) + + cmd = sys.argv[1] + args = sys.argv[2:] + token: str | None = None + if args and not args[0].startswith("http"): + token = args[0] + args = args[1:] + token = token or resolve_git_token() + + if cmd == "fetch-raw": + repo, ref, path = args[:3] + sys.stdout.buffer.write(fetch_raw_file(repo, ref, path, token)) + return + + if cmd == "resolve-tag": + print(resolve_common_tag(args, token)) + return + + sys.exit(f"Unknown command: {cmd}") + + +if __name__ == "__main__": + main() diff --git a/scripts/lib.sh b/scripts/lib.sh new file mode 100644 index 0000000..470fc48 --- /dev/null +++ b/scripts/lib.sh @@ -0,0 +1,256 @@ +#!/usr/bin/env bash +# Shared helpers for FromChat deployment installer. + +set -euo pipefail + +NC=$'\033[0m' +GRAY=$'\033[38;5;245m' +BLUE=$'\033[38;5;81m' +PURPLE=$'\033[38;5;141m' +RED=$'\033[38;5;203m' +LIME=$'\033[38;5;154m' +ORANGE=$'\033[38;5;208m' +YELLOW=$'\033[38;5;226m' +CHECK=$'\033[38;5;154m' +CROSS=$'\033[38;5;203m' + +info() { printf '%b%s%b\n' "$BLUE" "$*" "$NC"; } +success() { printf '%b✓ %s%b\n' "$LIME" "$*" "$NC"; } +warn() { printf '%b⚠ %s%b\n' "$YELLOW" "$*" "$NC"; } +error() { printf '%b✗ %s%b\n' "$RED" "$*" "$NC" >&2; } +step() { printf '\n%b▸ %s%b\n' "$PURPLE" "$*" "$NC"; } +prompt() { printf '%b%s%b ' "$ORANGE" "$*" "$NC" >&2; } + +die() { + error "$1" + exit "${2:-1}" +} + +require_root() { + if [[ "${EUID}" -ne 0 ]]; then + info "Elevating privileges with sudo…" + exec sudo -E bash "$0" "$@" + fi +} + +is_debian_based() { + [[ -f /etc/debian_version ]] || grep -qiE 'ubuntu|debian|mint|pop|elementary|raspbian' /etc/os-release 2>/dev/null +} + +require_debian() { + if ! is_debian_based; then + die "This installer supports Debian-based Linux only (Debian, Ubuntu, etc.)." + fi + success "Debian-based OS detected" +} + +docker_installed() { + command -v docker >/dev/null 2>&1 && docker info >/dev/null 2>&1 +} + +install_docker_if_needed() { + if docker_installed; then + success "Docker is already installed" + return 0 + fi + + warn "Docker is not installed or not running." + prompt "Install Docker using the official script? [Y/n]:" + local answer + IFS= read -r answer || true + answer="${answer:-Y}" + if [[ ! "${answer}" =~ ^[Yy] ]]; then + die "Docker is required. Install it manually: https://docs.docker.com/engine/install/" + fi + + step "Installing Docker…" + curl -fsSL https://get.docker.com | sh + systemctl enable --now docker 2>/dev/null || true + success "Docker installed" +} + +ensure_python_yaml() { + python3 - <<'PY' >/dev/null 2>&1 && return 0 +import yaml # noqa: F401 +PY + step "Installing python3-yaml…" + apt-get update -qq + DEBIAN_FRONTEND=noninteractive apt-get install -y -qq python3-yaml +} + +ensure_compose_plugin() { + if docker compose version >/dev/null 2>&1; then + success "Docker Compose plugin available" + return 0 + fi + step "Installing docker-compose-plugin…" + apt-get update -qq + DEBIAN_FRONTEND=noninteractive apt-get install -y -qq docker-compose-plugin +} + +repo_slug_from_url() { + local url="$1" + url="${url%.git}" + for prefix in "github.com/" "git.fromchat.ru/" "https://" "http://"; do + url="${url#*${prefix}}" + done + echo "${url%%/*}/${url##*/}" +} + +fetch_raw_file() { + local repo_url="$1" + local tag="$2" + local path="$3" + local dest="$4" + local script="${DEPLOYMENT_ROOT}/scripts/git_remote.py" + local token="${GIT_TOKEN:-${GITHUB_TOKEN:-}}" + if [[ -n "${token}" ]]; then + python3 "${script}" fetch-raw "${repo_url}" "${tag}" "${path}" "${token}" > "${dest}" + else + python3 "${script}" fetch-raw "${repo_url}" "${tag}" "${path}" > "${dest}" + fi +} + +get_latest_semver_tag() { + local repo_url="$1" + local token="${GIT_TOKEN:-${GITHUB_TOKEN:-}}" + if [[ -n "${token}" ]]; then + python3 "${DEPLOYMENT_ROOT}/scripts/resolve-tag.py" "${token}" "${repo_url}" + else + python3 "${DEPLOYMENT_ROOT}/scripts/resolve-tag.py" "${repo_url}" + fi +} + +select_components() { + if command -v whiptail >/dev/null 2>&1; then + local result + result="$(whiptail --title "FromChat components" --checklist \ + "Select components (Space toggles, Enter confirms)" 18 72 4 \ + backend "Backend (API, DB, LiveKit)" ON \ + frontend "Web frontend" ON \ + caddy "Caddy reverse proxy (TLS)" OFF \ + updater "Auto-update service" OFF \ + 3>&1 1>&2 2>&3)" || die "Component selection cancelled." + SELECTED=() + local item + for item in ${result}; do + SELECTED+=("${item//\"/}") + done + ((${#SELECTED[@]} > 0)) || die "Select at least one component." + return 0 + fi + + warn "whiptail not found; using text menu." + local -a options=(backend frontend caddy updater) + local -a labels=( + "Backend (API, DB, LiveKit)" + "Web frontend" + "Caddy reverse proxy" + "Auto-update service" + ) + SELECTED=() + local i choice + for i in "${!options[@]}"; do + prompt "Include ${labels[$i]}? [y/N]:" + IFS= read -r choice || true + if [[ "${choice:-}" =~ ^[Yy] ]]; then + SELECTED+=("${options[$i]}") + fi + done + ((${#SELECTED[@]} > 0)) || die "Select at least one component." +} + +component_selected() { + local needle="$1" + local item + for item in "${SELECTED[@]:-}"; do + [[ "${item}" == "${needle}" ]] && return 0 + done + return 1 +} + +wait_for_compose_healthy() { + local dir="$1" + local timeout="${2:-600}" + step "Waiting for services to become healthy (timeout ${timeout}s)…" + ( + cd "${dir}" + docker compose up -d --wait --timeout "${timeout}" + ) +} + +print_success_banner() { + local dir="$1" + printf '\n%b╔══════════════════════════════════════════════════╗%b\n' "$LIME" "$NC" + printf '%b║ FromChat server installed successfully! ║%b\n' "$LIME" "$NC" + printf '%b╚══════════════════════════════════════════════════╝%b\n\n' "$LIME" "$NC" + info "Install directory: ${dir}" + info "Version tag: ${FROMCHAT_VERSION}" + info "Manage stack: cd ${dir} && docker compose ps" + info "View logs: cd ${dir} && docker compose logs -f" + if component_selected caddy; then + info "Caddy config: ${dir}/Caddyfile" + fi +} + +# Merge component compose files into ${install_dir}/compose.yml (image: fromchat/*:). +# Optional local compose paths (skip remote fetch when set): +# LOCAL_BACKEND_COMPOSE, LOCAL_FRONTEND_COMPOSE +# Requires: DEPLOYMENT_ROOT, BACKEND_REPO, WEB_REPO (for remote fetch). +run_generate_compose() { + local install_dir="$1" + local components_csv="$2" + local tag="$3" + local tmp + tmp="$(mktemp -d)" + + step "Generating compose.yml for tag ${tag} (components: ${components_csv})…" + + local need_backend=false need_frontend=false + [[ ",${components_csv}," == *,backend,* ]] && need_backend=true + [[ ",${components_csv}," == *,frontend,* ]] && need_frontend=true + + if ${need_backend}; then + if [[ -n "${LOCAL_BACKEND_COMPOSE:-}" && -f "${LOCAL_BACKEND_COMPOSE}" ]]; then + cp -f "${LOCAL_BACKEND_COMPOSE}" "${tmp}/backend.compose.yml" + success "Backend compose.yml (local)" + else + fetch_raw_file "${BACKEND_REPO}" "${tag}" "compose.yml" \ + "${tmp}/backend.compose.yml" + success "Backend compose.yml" + fi + fi + if ${need_frontend}; then + if [[ -n "${LOCAL_FRONTEND_COMPOSE:-}" && -f "${LOCAL_FRONTEND_COMPOSE}" ]]; then + cp -f "${LOCAL_FRONTEND_COMPOSE}" "${tmp}/frontend.compose.yml" + success "Frontend compose.yml (local)" + else + fetch_raw_file "${WEB_REPO}" "${tag}" "compose.yml" \ + "${tmp}/frontend.compose.yml" + success "Frontend compose.yml" + fi + fi + + mkdir -p "${install_dir}/config" + cp -f "${DEPLOYMENT_ROOT}/config/livekit.yaml" "${install_dir}/config/livekit.yaml" + + local -a gen_args=( + python3 "${DEPLOYMENT_ROOT}/scripts/generate-compose.py" + --tag "${tag}" + --components "${components_csv}" + --output "${install_dir}/compose.yml" + ) + if ${need_backend}; then + gen_args+=(--backend-compose "${tmp}/backend.compose.yml") + fi + if ${need_frontend}; then + gen_args+=(--frontend-compose "${tmp}/frontend.compose.yml") + fi + if [[ ",${components_csv}," == *,caddy,* ]]; then + gen_args+=(--caddy-compose "${DEPLOYMENT_ROOT}/compose/caddy.compose.yml") + fi + "${gen_args[@]}" + + echo "${tag}" > "${install_dir}/.fromchat-version" + rm -rf "${tmp}" +} diff --git a/scripts/resolve-tag.py b/scripts/resolve-tag.py new file mode 100644 index 0000000..67e2b4e --- /dev/null +++ b/scripts/resolve-tag.py @@ -0,0 +1,25 @@ +#!/usr/bin/env python3 +"""Pick the latest semver tag present in all given repos (GitHub, Gitea fallback).""" +from __future__ import annotations + +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +from git_remote import resolve_common_tag # noqa: E402 + + +def main() -> None: + args = sys.argv[1:] + token: str | None = None + if args and not args[0].startswith("http"): + token = args[0] + args = args[1:] + token = token or resolve_git_token() + if not args: + sys.exit("usage: resolve-tag.py [TOKEN] REPO_URL ...") + print(resolve_common_tag(args, token)) + + +if __name__ == "__main__": + main() diff --git a/templates/Caddyfile b/templates/Caddyfile new file mode 100644 index 0000000..22f9dd0 --- /dev/null +++ b/templates/Caddyfile @@ -0,0 +1,32 @@ +{ + servers { + listener_wrappers { + proxy_protocol + tls + } + trusted_proxies static 127.0.0.1/32 ::1/128 + } + http_port 8080 + https_port 8443 +} + +# Replace example.com with your domain. Requires web service on the same compose network. +example.com { + reverse_proxy web:80 { + header_up X-Real-IP {remote_host} + } + + header { + X-Content-Type-Options "nosniff" + X-Frame-Options "DENY" + Referrer-Policy "strict-origin-when-cross-origin" + Strict-Transport-Security "max-age=31536000; includeSubDomains; preload" + } +} + +# API on same host (optional second site block) +api.example.com { + reverse_proxy main:8300 { + header_up X-Real-IP {remote_host} + } +} diff --git a/templates/fromchat.service b/templates/fromchat.service new file mode 100644 index 0000000..00c9125 --- /dev/null +++ b/templates/fromchat.service @@ -0,0 +1,33 @@ +[Unit] +Description=FromChat server +After=multi-user.target +Wants=network-online.target +After=network-online.target +StartLimitIntervalSec=60 +StartLimitBurst=3 + +[Service] +Type=simple +User=root +Group=root +ExecStart=/bin/bash -c "docker compose up --remove-orphans --force-recreate" +ExecStop=/bin/bash -c "docker compose down --remove-orphans" +WorkingDirectory=/home/fromchat/fromchat-server +Restart=always +RestartSec=10 +StartLimitBurst=3 + +# Security settings +NoNewPrivileges=true +PrivateTmp=true +ProtectSystem=strict +ReadWritePaths=/var/log + +# Logging +StandardOutput=journal +StandardError=journal +StandardInput=tty-force +SyslogIdentifier=fromchat + +[Install] +WantedBy=multi-user.target