From 1c2bcf513aeda73b89d77a9897dacd8cd96d8858 Mon Sep 17 00:00:00 2001 From: denis0001-dev Date: Wed, 15 Jul 2026 05:41:19 +0300 Subject: [PATCH] Fix caching --- scripts/deploy/compose_build.py | 18 +++- scripts/deploy/config.py | 14 --- scripts/deploy/main.py | 1 - scripts/deploy/transfer.py | 16 +++- scripts/deploy/updater.py | 70 +-------------- scripts/deploy/util.py | 10 ++- scripts/docker_inputs_hash.py | 150 ++++++++++++++++++++++++++++---- scripts/generate-compose.py | 6 +- 8 files changed, 172 insertions(+), 113 deletions(-) diff --git a/scripts/deploy/compose_build.py b/scripts/deploy/compose_build.py index eff7e8b..06d36eb 100644 --- a/scripts/deploy/compose_build.py +++ b/scripts/deploy/compose_build.py @@ -107,9 +107,6 @@ class ComposeBuildPhase: if not isinstance(build, dict): return "" parts: list[str] = [] - target = build.get("target") - if target: - parts.append(f"target={target}") args = build.get("args") if isinstance(args, dict): for key in sorted(args.keys()): @@ -119,12 +116,20 @@ class ComposeBuildPhase: parts.append(f"arg:{item}") return "\n".join(parts) + def _service_build_target(self, spec: dict) -> str | None: + build = spec.get("build") + if not isinstance(build, dict): + return None + target = build.get("target") + return str(target).strip() if target else None + def _service_input_hash(self, context: Path, dockerfile: Path, spec: dict) -> str: return compute_inputs_hash( context, dockerfile, hash_script=self._paths.input_hash_script, extra_material=self._service_build_identity(spec), + target=self._service_build_target(spec), ) def _partition_build_services( @@ -186,7 +191,10 @@ class ComposeBuildPhase: ui.success("Build skipped (no services to build)") return [] - need_build, _skip_build = self._partition_build_services(build_dir, services) + need_build, skip_build = self._partition_build_services(build_dir, services) + + if skip_build: + ui.substep(f"Build cache hit — skipping: {', '.join(skip_build)}") if need_build: ui.step(f"Building {len(need_build)} image(s) via docker compose") @@ -203,6 +211,8 @@ class ComposeBuildPhase: ui.error("docker compose build failed") sys.exit(1) ui.success(f"Build complete! {len(need_build)} image(s) built") + elif skip_build: + ui.success(f"Build skipped — {len(skip_build)} image(s) unchanged") compose = self.load_compose_json(build_dir, BUILD_COMPOSE_FILE) svc_map = compose.get("services") or {} diff --git a/scripts/deploy/config.py b/scripts/deploy/config.py index e4c27a4..6f76e19 100644 --- a/scripts/deploy/config.py +++ b/scripts/deploy/config.py @@ -17,7 +17,6 @@ from deploy.session_cache import ( save_cached_settings, settings_to_cache, ) -from deploy.util import read_env_file_value import deploy.ui as ui @@ -32,7 +31,6 @@ class DeploySettings: components: list[str] compose_components: list[str] tag: str - git_token: str | None paths: ProjectPaths @@ -197,17 +195,6 @@ def load_settings(paths: ProjectPaths, argv: list[str]) -> DeploySettings: platform_arch = docker_platform.split("/", 1)[-1] use_docker_build = bool(host_arch and host_arch == platform_arch) - git_token: str | None = None - prod_env = paths.deploy_env_source() - if prod_env: - git_token = read_env_file_value(prod_env, "UPDATER_TOKEN") or None - if not git_token: - git_token = ( - os.environ.get("UPDATER_TOKEN") - or os.environ.get("GIT_TOKEN") - or os.environ.get("GITHUB_TOKEN") - ) - if "backend" in components and not paths.backend_dir: sys.stderr.write( "Backend component selected but backend repo not found.\n" @@ -243,7 +230,6 @@ def load_settings(paths: ProjectPaths, argv: list[str]) -> DeploySettings: components=components, compose_components=stack, tag=tag, - git_token=git_token, paths=paths, ) diff --git a/scripts/deploy/main.py b/scripts/deploy/main.py index f792da4..12bc85f 100644 --- a/scripts/deploy/main.py +++ b/scripts/deploy/main.py @@ -216,7 +216,6 @@ def main() -> None: deploy_resolved, components=settings.components, paths=paths, - git_token=settings.git_token, ) if "backend" in stack: deploy_resolved = transfer.sync_firebase_cert(creds, deploy_resolved) diff --git a/scripts/deploy/transfer.py b/scripts/deploy/transfer.py index 01bdc40..ba87e96 100644 --- a/scripts/deploy/transfer.py +++ b/scripts/deploy/transfer.py @@ -14,6 +14,7 @@ from deploy.ssh_auth import SshCredentials, scp_argv, ssh_argv, ssh_common_optio from deploy.util import ( image_exists_locally, local_image_layer_fp, + read_cached_layer_fp, remote_image_layer_fp, write_image_cache_fields, ) @@ -188,16 +189,24 @@ class DeployTransfer: ) -> tuple[list[str], list[str]]: need: list[str] = [] skip: list[str] = [] + cache_root = self._paths.local_image_cache_dir for image in images: local_fp = local_image_layer_fp(image) if not local_fp: need.append(image) continue + + cached_local = read_cached_layer_fp(cache_root, image, "local") + cached_remote = read_cached_layer_fp(cache_root, image, "remote") + if cached_local == local_fp and cached_remote == local_fp: + skip.append(image) + continue + remote_fp = remote_image_layer_fp(creds.server, image) if remote_fp and remote_fp == local_fp: skip.append(image) write_image_cache_fields( - self._paths.local_image_cache_dir, + cache_root, image, **{ "local.layer.sha256": local_fp, @@ -211,8 +220,11 @@ class DeployTransfer: def pussh_images(self, creds: SshCredentials, images: list[str]) -> None: if not images: return - need, _skip = self._images_needing_transfer(creds, images) + need, skip = self._images_needing_transfer(creds, images) + if skip: + ui.substep(f"Transfer cache hit — skipping: {', '.join(skip)}") if not need: + ui.success(f"All {len(skip)} image(s) already on server") return ui.step("Transferring images to server") self.ensure_unregistry(creds) diff --git a/scripts/deploy/updater.py b/scripts/deploy/updater.py index 0a32f7b..00db407 100644 --- a/scripts/deploy/updater.py +++ b/scripts/deploy/updater.py @@ -2,85 +2,30 @@ from __future__ import annotations -import getpass -import os import shlex import subprocess from pathlib import Path from deploy.paths import ProjectPaths from deploy.ssh_auth import SshCredentials, ssh_argv, ssh_common_options -from deploy.util import read_env_file_value import deploy.ui as ui -DEFAULT_BACKEND_REPO = "https://github.com/fromchat-messenger/backend.git" -DEFAULT_WEB_REPO = "https://github.com/fromchat-messenger/web.git" -DEFAULT_APP_REPO = "https://github.com/fromchat-messenger/app.git" - - -def _repo_urls(paths: ProjectPaths) -> tuple[str, str, str]: - backend = os.environ.get("FROMCHAT_BACKEND_REPO", DEFAULT_BACKEND_REPO) - web = os.environ.get("FROMCHAT_WEB_REPO", DEFAULT_WEB_REPO) - app = os.environ.get("FROMCHAT_APP_REPO", DEFAULT_APP_REPO) - env_src = paths.deploy_env_source() - if env_src: - for key, current in ( - ("FROMCHAT_BACKEND_REPO", "backend"), - ("FROMCHAT_WEB_REPO", "web"), - ("FROMCHAT_APP_REPO", "app"), - ): - val = read_env_file_value(env_src, key) - if val: - if current == "backend": - backend = val - elif current == "web": - web = val - else: - app = val - return backend, web, app - - -def resolve_updater_token(explicit: str | None, paths: ProjectPaths) -> str: - if explicit: - return explicit - env_src = paths.deploy_env_source() - if env_src: - token = read_env_file_value(env_src, "UPDATER_TOKEN") - if token: - return token - for key in ("UPDATER_TOKEN", "GIT_TOKEN", "GITHUB_TOKEN"): - val = os.environ.get(key, "").strip() - if val: - return val - ui.step("Updater token (GitHub or Gitea)") - print(" GitHub: https://github.com/settings/tokens/new?scopes=read:packages,repo") - print(" Gitea: Settings → Applications → Generate New Token") - token = getpass.getpass(" Paste UPDATER_TOKEN (input hidden): ").strip() - if not token: - ui.error("UPDATER_TOKEN is required when updater is selected.") - raise SystemExit(1) - return token - def write_updater_env( dest: Path, *, deploy_path_resolved: str, components: list[str], - paths: ProjectPaths, ) -> None: - backend, web, app = _repo_urls(paths) - components_csv = ",".join(components) + components_csv = ",".join(c for c in components if c != "updater") dest.parent.mkdir(parents=True, exist_ok=True) dest.write_text( "\n".join( [ - f"BACKEND_REPO={backend}", - f"WEB_REPO={web}", - f"DEPLOYMENT_REPO={app}", f"COMPOSE_PROJECT_DIR={deploy_path_resolved}", f"FROMCHAT_COMPONENTS={components_csv}", "CHECK_INTERVAL_SECONDS=60", + "DOCKERHUB_NAMESPACE=fromchat", "", ] ), @@ -94,23 +39,14 @@ def setup_updater_remote( *, components: list[str], paths: ProjectPaths, - git_token: str | None, ) -> None: ui.step("Setting up auto-updater on server") - token = resolve_updater_token(git_token, paths) - if not token: - ui.error( - "UPDATER_TOKEN is missing. Add it to deployment/.env.prod " - "(e.g. npm run generate:env from backend, output ../deployment/.env.prod) before deploying with updater." - ) - raise SystemExit(1) staging_env = paths.staging_dir / "updater" / ".env" write_updater_env( staging_env, deploy_path_resolved=deploy_path_resolved, components=components, - paths=paths, ) remote_updater = f"{deploy_path_resolved}/updater" @@ -138,4 +74,4 @@ def setup_updater_remote( print(f" {line}") raise SystemExit(1) - ui.success("Updater env synced (included in compose stack)") + ui.success("Updater env synced (Docker Hub polling)") diff --git a/scripts/deploy/util.py b/scripts/deploy/util.py index 9ddefac..5356b36 100644 --- a/scripts/deploy/util.py +++ b/scripts/deploy/util.py @@ -38,6 +38,10 @@ def read_cached_input_hash(cache_root: Path, image_ref: str) -> str: return read_file_if_exists(image_cache_dir(cache_root, image_ref) / "input.sha256").strip() +def read_cached_layer_fp(cache_root: Path, image_ref: str, kind: str) -> str: + return read_file_if_exists(image_cache_dir(cache_root, image_ref) / f"{kind}.layer.sha256").strip() + + def write_image_cache_fields(cache_root: Path, image_ref: str, **fields: str) -> None: d = image_cache_dir(cache_root, image_ref) d.mkdir(parents=True, exist_ok=True) @@ -96,10 +100,14 @@ def compute_inputs_hash( hash_script: Path, python_exe: str | None = None, extra_material: str = "", + target: str | None = None, ) -> str: exe = python_exe or sys.executable + cmd = [exe, str(hash_script), "--context", str(context), "--dockerfile", str(dockerfile)] + if target: + cmd.extend(["--target", target]) p = subprocess.run( - [exe, str(hash_script), "--context", str(context), "--dockerfile", str(dockerfile)], + cmd, capture_output=True, text=True, ) diff --git a/scripts/docker_inputs_hash.py b/scripts/docker_inputs_hash.py index 838c395..7ab30b9 100644 --- a/scripts/docker_inputs_hash.py +++ b/scripts/docker_inputs_hash.py @@ -253,31 +253,129 @@ def _iter_files_under(path: Path) -> list[Path]: return files -def _collect_sources(context: Path, dockerfile_path: Path) -> list[Path]: +@dataclass(frozen=True) +class StageInfo: + name: str + parent: str | None + copies: tuple[CopyAdd, ...] + + +def _parse_from_stage(line: str, fallback_index: int) -> tuple[str, str | None]: + """Return (stage_name, parent_stage_or_none).""" + upper = line.upper() + if not upper.startswith("FROM "): + raise ValueError(line) + rest = line[5:].strip() + parts = rest.split() + if len(parts) >= 3 and parts[-2].upper() == "AS": + parent_token = " ".join(parts[:-2]).strip() + name = parts[-1] + elif len(parts) >= 1: + parent_token = parts[0] + name = f"stage{fallback_index}" + else: + name = f"stage{fallback_index}" + parent_token = "" + + parent: str | None + if not parent_token or parent_token.lower() in ( + "scratch", + ) or "/" in parent_token or ":" in parent_token: + # Image reference (python:3.12-slim) — not a stage alias. + parent = None + else: + parent = parent_token + return name, parent + + +def _parse_stages(logical: list[str]) -> list[StageInfo]: + stages: list[StageInfo] = [] + current_name: str | None = None + current_parent: str | None = None + current_copies: list[CopyAdd] = [] + fallback_index = 0 + + def flush() -> None: + nonlocal current_name, current_parent, current_copies + if current_name is None: + return + stages.append( + StageInfo( + name=current_name, + parent=current_parent, + copies=tuple(current_copies), + ) + ) + current_copies = [] + + for ln in logical: + upper = ln.lstrip().upper() + if upper.startswith("FROM "): + flush() + current_name, current_parent = _parse_from_stage(ln, fallback_index) + fallback_index += 1 + continue + parsed = _parse_copy_add(ln) + if parsed and not parsed.from_stage: + current_copies.append(parsed) + + flush() + return stages + + +def _stage_chain(stages: list[StageInfo], target: str | None) -> list[StageInfo]: + if not stages: + return [] + by_name = {s.name: s for s in stages} + if not target: + return list(stages) + + if target not in by_name: + return list(stages) + + chain: list[StageInfo] = [] + seen: set[str] = set() + name: str | None = target + while name and name not in seen: + seen.add(name) + stage = by_name.get(name) + if stage is None: + break + chain.append(stage) + name = stage.parent + chain.reverse() + return chain + + +def _collect_sources(context: Path, dockerfile_path: Path, *, target: str | None = None) -> list[Path]: text = _read_text(dockerfile_path) logical = _dockerfile_logical_lines(text) dockerignore_rules = _read_dockerignore_rules(context) + stages = _parse_stages(logical) + active_stages = _stage_chain(stages, target) paths: list[Path] = [] - for ln in logical: - parsed = _parse_copy_add(ln) - if not parsed: - continue - if parsed.from_stage: - continue + copies: list[CopyAdd] = [] + if active_stages: + for stage in active_stages: + copies.extend(stage.copies) + else: + for ln in logical: + parsed = _parse_copy_add(ln) + if parsed and not parsed.from_stage: + copies.append(parsed) + + for parsed in copies: for src in parsed.sources: if _looks_like_remote(src): continue if src.startswith("/"): - # Absolute COPY sources aren't valid for local context; ignore to avoid surprises. continue - # Docker allows ".", "./foo", etc. src_norm = src.lstrip("./") if src_norm == "": src_norm = "." if _is_glob(src_norm): - # Expand within context for root, _, filenames in os.walk(context): root_p = Path(root) rel_root = root_p.relative_to(context).as_posix() @@ -290,7 +388,6 @@ def _collect_sources(context: Path, dockerfile_path: Path) -> list[Path]: continue p = (context / src_norm).resolve() - # Ensure stays within context try: p.relative_to(context.resolve()) except Exception: @@ -304,13 +401,17 @@ def _collect_sources(context: Path, dockerfile_path: Path) -> list[Path]: continue paths.append(fp) - # Always include the Dockerfile itself (and preserve stable ordering via sort later) paths.append(dockerfile_path.resolve()) return paths -def compute_inputs_hash(context: Path, dockerfile_path: Path) -> str: - files = _collect_sources(context=context, dockerfile_path=dockerfile_path) +def compute_inputs_hash( + context: Path, + dockerfile_path: Path, + *, + target: str | None = None, +) -> str: + files = _collect_sources(context=context, dockerfile_path=dockerfile_path, target=target) # Deduplicate by resolved path uniq: dict[str, Path] = {} for p in files: @@ -344,11 +445,16 @@ def compute_inputs_hash(context: Path, dockerfile_path: Path) -> str: return h.hexdigest() -def compute_inputs_debug(context: Path, dockerfile_path: Path) -> tuple[str, list[tuple[str, str]]]: +def compute_inputs_debug( + context: Path, + dockerfile_path: Path, + *, + target: str | None = None, +) -> tuple[str, list[tuple[str, str]]]: """ Returns (inputs_hash, [(rel_path, sha256_of_file_contents), ...]) with dockerignore applied. """ - files = _collect_sources(context=context, dockerfile_path=dockerfile_path) + files = _collect_sources(context=context, dockerfile_path=dockerfile_path, target=target) uniq: dict[str, Path] = {} for p in files: uniq[str(p.resolve())] = p.resolve() @@ -371,13 +477,18 @@ def compute_inputs_debug(context: Path, dockerfile_path: Path) -> tuple[str, lis else: items.append((rp, "NONFILE")) - return compute_inputs_hash(context=context, dockerfile_path=dockerfile_path), items + return compute_inputs_hash(context=context, dockerfile_path=dockerfile_path, target=target), items def main() -> int: ap = argparse.ArgumentParser() ap.add_argument("--context", required=True, help="Build context directory") ap.add_argument("--dockerfile", required=True, help="Dockerfile path") + ap.add_argument( + "--target", + default="", + help="Dockerfile build target (multi-stage); only COPY steps for that stage chain are hashed", + ) ap.add_argument( "--debug-list", action="store_true", @@ -387,6 +498,7 @@ def main() -> int: context = Path(args.context).resolve() dockerfile = Path(args.dockerfile).resolve() + target = args.target.strip() or None if not context.exists() or not context.is_dir(): print(f"Context not found or not a directory: {context}", file=sys.stderr) @@ -396,12 +508,12 @@ def main() -> int: return 2 if args.debug_list: - h, items = compute_inputs_debug(context=context, dockerfile_path=dockerfile) + h, items = compute_inputs_debug(context=context, dockerfile_path=dockerfile, target=target) for rp, sh in items: print(f"{rp}|{sh}", file=sys.stderr) print(h) else: - print(compute_inputs_hash(context=context, dockerfile_path=dockerfile)) + print(compute_inputs_hash(context=context, dockerfile_path=dockerfile, target=target)) return 0 diff --git a/scripts/generate-compose.py b/scripts/generate-compose.py index 0740475..04e9e8f 100644 --- a/scripts/generate-compose.py +++ b/scripts/generate-compose.py @@ -172,11 +172,7 @@ def patch_updater_service(service: dict[str, Any], *, keep_build: bool) -> None: service.pop("volumes", None) return service["env_file"] = ["updater/.env", ".env"] - environment = service.get("environment") - if not isinstance(environment, dict): - environment = {} - environment["UPDATER_TOKEN"] = "${UPDATER_TOKEN}" - service["environment"] = environment + service.pop("environment", None) service["volumes"] = [ "./compose.yml:/fromchat/compose.yml:rw", "/var/run/docker.sock:/var/run/docker.sock",