Fix caching

This commit is contained in:
2026-07-15 05:41:19 +03:00
Unverified
parent b647160d3f
commit 1c2bcf513a
8 changed files with 172 additions and 113 deletions
+14 -4
View File
@@ -107,9 +107,6 @@ class ComposeBuildPhase:
if not isinstance(build, dict):
return ""
parts: list[str] = []
target = build.get("target")
if target:
parts.append(f"target={target}")
args = build.get("args")
if isinstance(args, dict):
for key in sorted(args.keys()):
@@ -119,12 +116,20 @@ class ComposeBuildPhase:
parts.append(f"arg:{item}")
return "\n".join(parts)
def _service_build_target(self, spec: dict) -> str | None:
build = spec.get("build")
if not isinstance(build, dict):
return None
target = build.get("target")
return str(target).strip() if target else None
def _service_input_hash(self, context: Path, dockerfile: Path, spec: dict) -> str:
return compute_inputs_hash(
context,
dockerfile,
hash_script=self._paths.input_hash_script,
extra_material=self._service_build_identity(spec),
target=self._service_build_target(spec),
)
def _partition_build_services(
@@ -186,7 +191,10 @@ class ComposeBuildPhase:
ui.success("Build skipped (no services to build)")
return []
need_build, _skip_build = self._partition_build_services(build_dir, services)
need_build, skip_build = self._partition_build_services(build_dir, services)
if skip_build:
ui.substep(f"Build cache hit — skipping: {', '.join(skip_build)}")
if need_build:
ui.step(f"Building {len(need_build)} image(s) via docker compose")
@@ -203,6 +211,8 @@ class ComposeBuildPhase:
ui.error("docker compose build failed")
sys.exit(1)
ui.success(f"Build complete! {len(need_build)} image(s) built")
elif skip_build:
ui.success(f"Build skipped — {len(skip_build)} image(s) unchanged")
compose = self.load_compose_json(build_dir, BUILD_COMPOSE_FILE)
svc_map = compose.get("services") or {}
-14
View File
@@ -17,7 +17,6 @@ from deploy.session_cache import (
save_cached_settings,
settings_to_cache,
)
from deploy.util import read_env_file_value
import deploy.ui as ui
@@ -32,7 +31,6 @@ class DeploySettings:
components: list[str]
compose_components: list[str]
tag: str
git_token: str | None
paths: ProjectPaths
@@ -197,17 +195,6 @@ def load_settings(paths: ProjectPaths, argv: list[str]) -> DeploySettings:
platform_arch = docker_platform.split("/", 1)[-1]
use_docker_build = bool(host_arch and host_arch == platform_arch)
git_token: str | None = None
prod_env = paths.deploy_env_source()
if prod_env:
git_token = read_env_file_value(prod_env, "UPDATER_TOKEN") or None
if not git_token:
git_token = (
os.environ.get("UPDATER_TOKEN")
or os.environ.get("GIT_TOKEN")
or os.environ.get("GITHUB_TOKEN")
)
if "backend" in components and not paths.backend_dir:
sys.stderr.write(
"Backend component selected but backend repo not found.\n"
@@ -243,7 +230,6 @@ def load_settings(paths: ProjectPaths, argv: list[str]) -> DeploySettings:
components=components,
compose_components=stack,
tag=tag,
git_token=git_token,
paths=paths,
)
-1
View File
@@ -216,7 +216,6 @@ def main() -> None:
deploy_resolved,
components=settings.components,
paths=paths,
git_token=settings.git_token,
)
if "backend" in stack:
deploy_resolved = transfer.sync_firebase_cert(creds, deploy_resolved)
+14 -2
View File
@@ -14,6 +14,7 @@ from deploy.ssh_auth import SshCredentials, scp_argv, ssh_argv, ssh_common_optio
from deploy.util import (
image_exists_locally,
local_image_layer_fp,
read_cached_layer_fp,
remote_image_layer_fp,
write_image_cache_fields,
)
@@ -188,16 +189,24 @@ class DeployTransfer:
) -> tuple[list[str], list[str]]:
need: list[str] = []
skip: list[str] = []
cache_root = self._paths.local_image_cache_dir
for image in images:
local_fp = local_image_layer_fp(image)
if not local_fp:
need.append(image)
continue
cached_local = read_cached_layer_fp(cache_root, image, "local")
cached_remote = read_cached_layer_fp(cache_root, image, "remote")
if cached_local == local_fp and cached_remote == local_fp:
skip.append(image)
continue
remote_fp = remote_image_layer_fp(creds.server, image)
if remote_fp and remote_fp == local_fp:
skip.append(image)
write_image_cache_fields(
self._paths.local_image_cache_dir,
cache_root,
image,
**{
"local.layer.sha256": local_fp,
@@ -211,8 +220,11 @@ class DeployTransfer:
def pussh_images(self, creds: SshCredentials, images: list[str]) -> None:
if not images:
return
need, _skip = self._images_needing_transfer(creds, images)
need, skip = self._images_needing_transfer(creds, images)
if skip:
ui.substep(f"Transfer cache hit — skipping: {', '.join(skip)}")
if not need:
ui.success(f"All {len(skip)} image(s) already on server")
return
ui.step("Transferring images to server")
self.ensure_unregistry(creds)
+3 -67
View File
@@ -2,85 +2,30 @@
from __future__ import annotations
import getpass
import os
import shlex
import subprocess
from pathlib import Path
from deploy.paths import ProjectPaths
from deploy.ssh_auth import SshCredentials, ssh_argv, ssh_common_options
from deploy.util import read_env_file_value
import deploy.ui as ui
DEFAULT_BACKEND_REPO = "https://github.com/fromchat-messenger/backend.git"
DEFAULT_WEB_REPO = "https://github.com/fromchat-messenger/web.git"
DEFAULT_APP_REPO = "https://github.com/fromchat-messenger/app.git"
def _repo_urls(paths: ProjectPaths) -> tuple[str, str, str]:
backend = os.environ.get("FROMCHAT_BACKEND_REPO", DEFAULT_BACKEND_REPO)
web = os.environ.get("FROMCHAT_WEB_REPO", DEFAULT_WEB_REPO)
app = os.environ.get("FROMCHAT_APP_REPO", DEFAULT_APP_REPO)
env_src = paths.deploy_env_source()
if env_src:
for key, current in (
("FROMCHAT_BACKEND_REPO", "backend"),
("FROMCHAT_WEB_REPO", "web"),
("FROMCHAT_APP_REPO", "app"),
):
val = read_env_file_value(env_src, key)
if val:
if current == "backend":
backend = val
elif current == "web":
web = val
else:
app = val
return backend, web, app
def resolve_updater_token(explicit: str | None, paths: ProjectPaths) -> str:
if explicit:
return explicit
env_src = paths.deploy_env_source()
if env_src:
token = read_env_file_value(env_src, "UPDATER_TOKEN")
if token:
return token
for key in ("UPDATER_TOKEN", "GIT_TOKEN", "GITHUB_TOKEN"):
val = os.environ.get(key, "").strip()
if val:
return val
ui.step("Updater token (GitHub or Gitea)")
print(" GitHub: https://github.com/settings/tokens/new?scopes=read:packages,repo")
print(" Gitea: Settings → Applications → Generate New Token")
token = getpass.getpass(" Paste UPDATER_TOKEN (input hidden): ").strip()
if not token:
ui.error("UPDATER_TOKEN is required when updater is selected.")
raise SystemExit(1)
return token
def write_updater_env(
dest: Path,
*,
deploy_path_resolved: str,
components: list[str],
paths: ProjectPaths,
) -> None:
backend, web, app = _repo_urls(paths)
components_csv = ",".join(components)
components_csv = ",".join(c for c in components if c != "updater")
dest.parent.mkdir(parents=True, exist_ok=True)
dest.write_text(
"\n".join(
[
f"BACKEND_REPO={backend}",
f"WEB_REPO={web}",
f"DEPLOYMENT_REPO={app}",
f"COMPOSE_PROJECT_DIR={deploy_path_resolved}",
f"FROMCHAT_COMPONENTS={components_csv}",
"CHECK_INTERVAL_SECONDS=60",
"DOCKERHUB_NAMESPACE=fromchat",
"",
]
),
@@ -94,23 +39,14 @@ def setup_updater_remote(
*,
components: list[str],
paths: ProjectPaths,
git_token: str | None,
) -> None:
ui.step("Setting up auto-updater on server")
token = resolve_updater_token(git_token, paths)
if not token:
ui.error(
"UPDATER_TOKEN is missing. Add it to deployment/.env.prod "
"(e.g. npm run generate:env from backend, output ../deployment/.env.prod) before deploying with updater."
)
raise SystemExit(1)
staging_env = paths.staging_dir / "updater" / ".env"
write_updater_env(
staging_env,
deploy_path_resolved=deploy_path_resolved,
components=components,
paths=paths,
)
remote_updater = f"{deploy_path_resolved}/updater"
@@ -138,4 +74,4 @@ def setup_updater_remote(
print(f" {line}")
raise SystemExit(1)
ui.success("Updater env synced (included in compose stack)")
ui.success("Updater env synced (Docker Hub polling)")
+9 -1
View File
@@ -38,6 +38,10 @@ def read_cached_input_hash(cache_root: Path, image_ref: str) -> str:
return read_file_if_exists(image_cache_dir(cache_root, image_ref) / "input.sha256").strip()
def read_cached_layer_fp(cache_root: Path, image_ref: str, kind: str) -> str:
return read_file_if_exists(image_cache_dir(cache_root, image_ref) / f"{kind}.layer.sha256").strip()
def write_image_cache_fields(cache_root: Path, image_ref: str, **fields: str) -> None:
d = image_cache_dir(cache_root, image_ref)
d.mkdir(parents=True, exist_ok=True)
@@ -96,10 +100,14 @@ def compute_inputs_hash(
hash_script: Path,
python_exe: str | None = None,
extra_material: str = "",
target: str | None = None,
) -> str:
exe = python_exe or sys.executable
cmd = [exe, str(hash_script), "--context", str(context), "--dockerfile", str(dockerfile)]
if target:
cmd.extend(["--target", target])
p = subprocess.run(
[exe, str(hash_script), "--context", str(context), "--dockerfile", str(dockerfile)],
cmd,
capture_output=True,
text=True,
)
+131 -19
View File
@@ -253,31 +253,129 @@ def _iter_files_under(path: Path) -> list[Path]:
return files
def _collect_sources(context: Path, dockerfile_path: Path) -> list[Path]:
@dataclass(frozen=True)
class StageInfo:
name: str
parent: str | None
copies: tuple[CopyAdd, ...]
def _parse_from_stage(line: str, fallback_index: int) -> tuple[str, str | None]:
"""Return (stage_name, parent_stage_or_none)."""
upper = line.upper()
if not upper.startswith("FROM "):
raise ValueError(line)
rest = line[5:].strip()
parts = rest.split()
if len(parts) >= 3 and parts[-2].upper() == "AS":
parent_token = " ".join(parts[:-2]).strip()
name = parts[-1]
elif len(parts) >= 1:
parent_token = parts[0]
name = f"stage{fallback_index}"
else:
name = f"stage{fallback_index}"
parent_token = ""
parent: str | None
if not parent_token or parent_token.lower() in (
"scratch",
) or "/" in parent_token or ":" in parent_token:
# Image reference (python:3.12-slim) — not a stage alias.
parent = None
else:
parent = parent_token
return name, parent
def _parse_stages(logical: list[str]) -> list[StageInfo]:
stages: list[StageInfo] = []
current_name: str | None = None
current_parent: str | None = None
current_copies: list[CopyAdd] = []
fallback_index = 0
def flush() -> None:
nonlocal current_name, current_parent, current_copies
if current_name is None:
return
stages.append(
StageInfo(
name=current_name,
parent=current_parent,
copies=tuple(current_copies),
)
)
current_copies = []
for ln in logical:
upper = ln.lstrip().upper()
if upper.startswith("FROM "):
flush()
current_name, current_parent = _parse_from_stage(ln, fallback_index)
fallback_index += 1
continue
parsed = _parse_copy_add(ln)
if parsed and not parsed.from_stage:
current_copies.append(parsed)
flush()
return stages
def _stage_chain(stages: list[StageInfo], target: str | None) -> list[StageInfo]:
if not stages:
return []
by_name = {s.name: s for s in stages}
if not target:
return list(stages)
if target not in by_name:
return list(stages)
chain: list[StageInfo] = []
seen: set[str] = set()
name: str | None = target
while name and name not in seen:
seen.add(name)
stage = by_name.get(name)
if stage is None:
break
chain.append(stage)
name = stage.parent
chain.reverse()
return chain
def _collect_sources(context: Path, dockerfile_path: Path, *, target: str | None = None) -> list[Path]:
text = _read_text(dockerfile_path)
logical = _dockerfile_logical_lines(text)
dockerignore_rules = _read_dockerignore_rules(context)
stages = _parse_stages(logical)
active_stages = _stage_chain(stages, target)
paths: list[Path] = []
for ln in logical:
parsed = _parse_copy_add(ln)
if not parsed:
continue
if parsed.from_stage:
continue
copies: list[CopyAdd] = []
if active_stages:
for stage in active_stages:
copies.extend(stage.copies)
else:
for ln in logical:
parsed = _parse_copy_add(ln)
if parsed and not parsed.from_stage:
copies.append(parsed)
for parsed in copies:
for src in parsed.sources:
if _looks_like_remote(src):
continue
if src.startswith("/"):
# Absolute COPY sources aren't valid for local context; ignore to avoid surprises.
continue
# Docker allows ".", "./foo", etc.
src_norm = src.lstrip("./")
if src_norm == "":
src_norm = "."
if _is_glob(src_norm):
# Expand within context
for root, _, filenames in os.walk(context):
root_p = Path(root)
rel_root = root_p.relative_to(context).as_posix()
@@ -290,7 +388,6 @@ def _collect_sources(context: Path, dockerfile_path: Path) -> list[Path]:
continue
p = (context / src_norm).resolve()
# Ensure stays within context
try:
p.relative_to(context.resolve())
except Exception:
@@ -304,13 +401,17 @@ def _collect_sources(context: Path, dockerfile_path: Path) -> list[Path]:
continue
paths.append(fp)
# Always include the Dockerfile itself (and preserve stable ordering via sort later)
paths.append(dockerfile_path.resolve())
return paths
def compute_inputs_hash(context: Path, dockerfile_path: Path) -> str:
files = _collect_sources(context=context, dockerfile_path=dockerfile_path)
def compute_inputs_hash(
context: Path,
dockerfile_path: Path,
*,
target: str | None = None,
) -> str:
files = _collect_sources(context=context, dockerfile_path=dockerfile_path, target=target)
# Deduplicate by resolved path
uniq: dict[str, Path] = {}
for p in files:
@@ -344,11 +445,16 @@ def compute_inputs_hash(context: Path, dockerfile_path: Path) -> str:
return h.hexdigest()
def compute_inputs_debug(context: Path, dockerfile_path: Path) -> tuple[str, list[tuple[str, str]]]:
def compute_inputs_debug(
context: Path,
dockerfile_path: Path,
*,
target: str | None = None,
) -> tuple[str, list[tuple[str, str]]]:
"""
Returns (inputs_hash, [(rel_path, sha256_of_file_contents), ...]) with dockerignore applied.
"""
files = _collect_sources(context=context, dockerfile_path=dockerfile_path)
files = _collect_sources(context=context, dockerfile_path=dockerfile_path, target=target)
uniq: dict[str, Path] = {}
for p in files:
uniq[str(p.resolve())] = p.resolve()
@@ -371,13 +477,18 @@ def compute_inputs_debug(context: Path, dockerfile_path: Path) -> tuple[str, lis
else:
items.append((rp, "NONFILE"))
return compute_inputs_hash(context=context, dockerfile_path=dockerfile_path), items
return compute_inputs_hash(context=context, dockerfile_path=dockerfile_path, target=target), items
def main() -> int:
ap = argparse.ArgumentParser()
ap.add_argument("--context", required=True, help="Build context directory")
ap.add_argument("--dockerfile", required=True, help="Dockerfile path")
ap.add_argument(
"--target",
default="",
help="Dockerfile build target (multi-stage); only COPY steps for that stage chain are hashed",
)
ap.add_argument(
"--debug-list",
action="store_true",
@@ -387,6 +498,7 @@ def main() -> int:
context = Path(args.context).resolve()
dockerfile = Path(args.dockerfile).resolve()
target = args.target.strip() or None
if not context.exists() or not context.is_dir():
print(f"Context not found or not a directory: {context}", file=sys.stderr)
@@ -396,12 +508,12 @@ def main() -> int:
return 2
if args.debug_list:
h, items = compute_inputs_debug(context=context, dockerfile_path=dockerfile)
h, items = compute_inputs_debug(context=context, dockerfile_path=dockerfile, target=target)
for rp, sh in items:
print(f"{rp}|{sh}", file=sys.stderr)
print(h)
else:
print(compute_inputs_hash(context=context, dockerfile_path=dockerfile))
print(compute_inputs_hash(context=context, dockerfile_path=dockerfile, target=target))
return 0
+1 -5
View File
@@ -172,11 +172,7 @@ def patch_updater_service(service: dict[str, Any], *, keep_build: bool) -> None:
service.pop("volumes", None)
return
service["env_file"] = ["updater/.env", ".env"]
environment = service.get("environment")
if not isinstance(environment, dict):
environment = {}
environment["UPDATER_TOKEN"] = "${UPDATER_TOKEN}"
service["environment"] = environment
service.pop("environment", None)
service["volumes"] = [
"./compose.yml:/fromchat/compose.yml:rw",
"/var/run/docker.sock:/var/run/docker.sock",