mirror of
https://github.com/fromchat-messenger/deployment.git
synced 2026-11-16 20:41:55 +03:00
Fix caching
This commit is contained in:
@@ -107,9 +107,6 @@ class ComposeBuildPhase:
|
||||
if not isinstance(build, dict):
|
||||
return ""
|
||||
parts: list[str] = []
|
||||
target = build.get("target")
|
||||
if target:
|
||||
parts.append(f"target={target}")
|
||||
args = build.get("args")
|
||||
if isinstance(args, dict):
|
||||
for key in sorted(args.keys()):
|
||||
@@ -119,12 +116,20 @@ class ComposeBuildPhase:
|
||||
parts.append(f"arg:{item}")
|
||||
return "\n".join(parts)
|
||||
|
||||
def _service_build_target(self, spec: dict) -> str | None:
|
||||
build = spec.get("build")
|
||||
if not isinstance(build, dict):
|
||||
return None
|
||||
target = build.get("target")
|
||||
return str(target).strip() if target else None
|
||||
|
||||
def _service_input_hash(self, context: Path, dockerfile: Path, spec: dict) -> str:
|
||||
return compute_inputs_hash(
|
||||
context,
|
||||
dockerfile,
|
||||
hash_script=self._paths.input_hash_script,
|
||||
extra_material=self._service_build_identity(spec),
|
||||
target=self._service_build_target(spec),
|
||||
)
|
||||
|
||||
def _partition_build_services(
|
||||
@@ -186,7 +191,10 @@ class ComposeBuildPhase:
|
||||
ui.success("Build skipped (no services to build)")
|
||||
return []
|
||||
|
||||
need_build, _skip_build = self._partition_build_services(build_dir, services)
|
||||
need_build, skip_build = self._partition_build_services(build_dir, services)
|
||||
|
||||
if skip_build:
|
||||
ui.substep(f"Build cache hit — skipping: {', '.join(skip_build)}")
|
||||
|
||||
if need_build:
|
||||
ui.step(f"Building {len(need_build)} image(s) via docker compose")
|
||||
@@ -203,6 +211,8 @@ class ComposeBuildPhase:
|
||||
ui.error("docker compose build failed")
|
||||
sys.exit(1)
|
||||
ui.success(f"Build complete! {len(need_build)} image(s) built")
|
||||
elif skip_build:
|
||||
ui.success(f"Build skipped — {len(skip_build)} image(s) unchanged")
|
||||
|
||||
compose = self.load_compose_json(build_dir, BUILD_COMPOSE_FILE)
|
||||
svc_map = compose.get("services") or {}
|
||||
|
||||
@@ -17,7 +17,6 @@ from deploy.session_cache import (
|
||||
save_cached_settings,
|
||||
settings_to_cache,
|
||||
)
|
||||
from deploy.util import read_env_file_value
|
||||
import deploy.ui as ui
|
||||
|
||||
|
||||
@@ -32,7 +31,6 @@ class DeploySettings:
|
||||
components: list[str]
|
||||
compose_components: list[str]
|
||||
tag: str
|
||||
git_token: str | None
|
||||
paths: ProjectPaths
|
||||
|
||||
|
||||
@@ -197,17 +195,6 @@ def load_settings(paths: ProjectPaths, argv: list[str]) -> DeploySettings:
|
||||
platform_arch = docker_platform.split("/", 1)[-1]
|
||||
use_docker_build = bool(host_arch and host_arch == platform_arch)
|
||||
|
||||
git_token: str | None = None
|
||||
prod_env = paths.deploy_env_source()
|
||||
if prod_env:
|
||||
git_token = read_env_file_value(prod_env, "UPDATER_TOKEN") or None
|
||||
if not git_token:
|
||||
git_token = (
|
||||
os.environ.get("UPDATER_TOKEN")
|
||||
or os.environ.get("GIT_TOKEN")
|
||||
or os.environ.get("GITHUB_TOKEN")
|
||||
)
|
||||
|
||||
if "backend" in components and not paths.backend_dir:
|
||||
sys.stderr.write(
|
||||
"Backend component selected but backend repo not found.\n"
|
||||
@@ -243,7 +230,6 @@ def load_settings(paths: ProjectPaths, argv: list[str]) -> DeploySettings:
|
||||
components=components,
|
||||
compose_components=stack,
|
||||
tag=tag,
|
||||
git_token=git_token,
|
||||
paths=paths,
|
||||
)
|
||||
|
||||
|
||||
@@ -216,7 +216,6 @@ def main() -> None:
|
||||
deploy_resolved,
|
||||
components=settings.components,
|
||||
paths=paths,
|
||||
git_token=settings.git_token,
|
||||
)
|
||||
if "backend" in stack:
|
||||
deploy_resolved = transfer.sync_firebase_cert(creds, deploy_resolved)
|
||||
|
||||
@@ -14,6 +14,7 @@ from deploy.ssh_auth import SshCredentials, scp_argv, ssh_argv, ssh_common_optio
|
||||
from deploy.util import (
|
||||
image_exists_locally,
|
||||
local_image_layer_fp,
|
||||
read_cached_layer_fp,
|
||||
remote_image_layer_fp,
|
||||
write_image_cache_fields,
|
||||
)
|
||||
@@ -188,16 +189,24 @@ class DeployTransfer:
|
||||
) -> tuple[list[str], list[str]]:
|
||||
need: list[str] = []
|
||||
skip: list[str] = []
|
||||
cache_root = self._paths.local_image_cache_dir
|
||||
for image in images:
|
||||
local_fp = local_image_layer_fp(image)
|
||||
if not local_fp:
|
||||
need.append(image)
|
||||
continue
|
||||
|
||||
cached_local = read_cached_layer_fp(cache_root, image, "local")
|
||||
cached_remote = read_cached_layer_fp(cache_root, image, "remote")
|
||||
if cached_local == local_fp and cached_remote == local_fp:
|
||||
skip.append(image)
|
||||
continue
|
||||
|
||||
remote_fp = remote_image_layer_fp(creds.server, image)
|
||||
if remote_fp and remote_fp == local_fp:
|
||||
skip.append(image)
|
||||
write_image_cache_fields(
|
||||
self._paths.local_image_cache_dir,
|
||||
cache_root,
|
||||
image,
|
||||
**{
|
||||
"local.layer.sha256": local_fp,
|
||||
@@ -211,8 +220,11 @@ class DeployTransfer:
|
||||
def pussh_images(self, creds: SshCredentials, images: list[str]) -> None:
|
||||
if not images:
|
||||
return
|
||||
need, _skip = self._images_needing_transfer(creds, images)
|
||||
need, skip = self._images_needing_transfer(creds, images)
|
||||
if skip:
|
||||
ui.substep(f"Transfer cache hit — skipping: {', '.join(skip)}")
|
||||
if not need:
|
||||
ui.success(f"All {len(skip)} image(s) already on server")
|
||||
return
|
||||
ui.step("Transferring images to server")
|
||||
self.ensure_unregistry(creds)
|
||||
|
||||
@@ -2,85 +2,30 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import getpass
|
||||
import os
|
||||
import shlex
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
|
||||
from deploy.paths import ProjectPaths
|
||||
from deploy.ssh_auth import SshCredentials, ssh_argv, ssh_common_options
|
||||
from deploy.util import read_env_file_value
|
||||
import deploy.ui as ui
|
||||
|
||||
DEFAULT_BACKEND_REPO = "https://github.com/fromchat-messenger/backend.git"
|
||||
DEFAULT_WEB_REPO = "https://github.com/fromchat-messenger/web.git"
|
||||
DEFAULT_APP_REPO = "https://github.com/fromchat-messenger/app.git"
|
||||
|
||||
|
||||
def _repo_urls(paths: ProjectPaths) -> tuple[str, str, str]:
|
||||
backend = os.environ.get("FROMCHAT_BACKEND_REPO", DEFAULT_BACKEND_REPO)
|
||||
web = os.environ.get("FROMCHAT_WEB_REPO", DEFAULT_WEB_REPO)
|
||||
app = os.environ.get("FROMCHAT_APP_REPO", DEFAULT_APP_REPO)
|
||||
env_src = paths.deploy_env_source()
|
||||
if env_src:
|
||||
for key, current in (
|
||||
("FROMCHAT_BACKEND_REPO", "backend"),
|
||||
("FROMCHAT_WEB_REPO", "web"),
|
||||
("FROMCHAT_APP_REPO", "app"),
|
||||
):
|
||||
val = read_env_file_value(env_src, key)
|
||||
if val:
|
||||
if current == "backend":
|
||||
backend = val
|
||||
elif current == "web":
|
||||
web = val
|
||||
else:
|
||||
app = val
|
||||
return backend, web, app
|
||||
|
||||
|
||||
def resolve_updater_token(explicit: str | None, paths: ProjectPaths) -> str:
|
||||
if explicit:
|
||||
return explicit
|
||||
env_src = paths.deploy_env_source()
|
||||
if env_src:
|
||||
token = read_env_file_value(env_src, "UPDATER_TOKEN")
|
||||
if token:
|
||||
return token
|
||||
for key in ("UPDATER_TOKEN", "GIT_TOKEN", "GITHUB_TOKEN"):
|
||||
val = os.environ.get(key, "").strip()
|
||||
if val:
|
||||
return val
|
||||
ui.step("Updater token (GitHub or Gitea)")
|
||||
print(" GitHub: https://github.com/settings/tokens/new?scopes=read:packages,repo")
|
||||
print(" Gitea: Settings → Applications → Generate New Token")
|
||||
token = getpass.getpass(" Paste UPDATER_TOKEN (input hidden): ").strip()
|
||||
if not token:
|
||||
ui.error("UPDATER_TOKEN is required when updater is selected.")
|
||||
raise SystemExit(1)
|
||||
return token
|
||||
|
||||
|
||||
def write_updater_env(
|
||||
dest: Path,
|
||||
*,
|
||||
deploy_path_resolved: str,
|
||||
components: list[str],
|
||||
paths: ProjectPaths,
|
||||
) -> None:
|
||||
backend, web, app = _repo_urls(paths)
|
||||
components_csv = ",".join(components)
|
||||
components_csv = ",".join(c for c in components if c != "updater")
|
||||
dest.parent.mkdir(parents=True, exist_ok=True)
|
||||
dest.write_text(
|
||||
"\n".join(
|
||||
[
|
||||
f"BACKEND_REPO={backend}",
|
||||
f"WEB_REPO={web}",
|
||||
f"DEPLOYMENT_REPO={app}",
|
||||
f"COMPOSE_PROJECT_DIR={deploy_path_resolved}",
|
||||
f"FROMCHAT_COMPONENTS={components_csv}",
|
||||
"CHECK_INTERVAL_SECONDS=60",
|
||||
"DOCKERHUB_NAMESPACE=fromchat",
|
||||
"",
|
||||
]
|
||||
),
|
||||
@@ -94,23 +39,14 @@ def setup_updater_remote(
|
||||
*,
|
||||
components: list[str],
|
||||
paths: ProjectPaths,
|
||||
git_token: str | None,
|
||||
) -> None:
|
||||
ui.step("Setting up auto-updater on server")
|
||||
token = resolve_updater_token(git_token, paths)
|
||||
if not token:
|
||||
ui.error(
|
||||
"UPDATER_TOKEN is missing. Add it to deployment/.env.prod "
|
||||
"(e.g. npm run generate:env from backend, output ../deployment/.env.prod) before deploying with updater."
|
||||
)
|
||||
raise SystemExit(1)
|
||||
|
||||
staging_env = paths.staging_dir / "updater" / ".env"
|
||||
write_updater_env(
|
||||
staging_env,
|
||||
deploy_path_resolved=deploy_path_resolved,
|
||||
components=components,
|
||||
paths=paths,
|
||||
)
|
||||
|
||||
remote_updater = f"{deploy_path_resolved}/updater"
|
||||
@@ -138,4 +74,4 @@ def setup_updater_remote(
|
||||
print(f" {line}")
|
||||
raise SystemExit(1)
|
||||
|
||||
ui.success("Updater env synced (included in compose stack)")
|
||||
ui.success("Updater env synced (Docker Hub polling)")
|
||||
|
||||
@@ -38,6 +38,10 @@ def read_cached_input_hash(cache_root: Path, image_ref: str) -> str:
|
||||
return read_file_if_exists(image_cache_dir(cache_root, image_ref) / "input.sha256").strip()
|
||||
|
||||
|
||||
def read_cached_layer_fp(cache_root: Path, image_ref: str, kind: str) -> str:
|
||||
return read_file_if_exists(image_cache_dir(cache_root, image_ref) / f"{kind}.layer.sha256").strip()
|
||||
|
||||
|
||||
def write_image_cache_fields(cache_root: Path, image_ref: str, **fields: str) -> None:
|
||||
d = image_cache_dir(cache_root, image_ref)
|
||||
d.mkdir(parents=True, exist_ok=True)
|
||||
@@ -96,10 +100,14 @@ def compute_inputs_hash(
|
||||
hash_script: Path,
|
||||
python_exe: str | None = None,
|
||||
extra_material: str = "",
|
||||
target: str | None = None,
|
||||
) -> str:
|
||||
exe = python_exe or sys.executable
|
||||
cmd = [exe, str(hash_script), "--context", str(context), "--dockerfile", str(dockerfile)]
|
||||
if target:
|
||||
cmd.extend(["--target", target])
|
||||
p = subprocess.run(
|
||||
[exe, str(hash_script), "--context", str(context), "--dockerfile", str(dockerfile)],
|
||||
cmd,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
|
||||
+131
-19
@@ -253,31 +253,129 @@ def _iter_files_under(path: Path) -> list[Path]:
|
||||
return files
|
||||
|
||||
|
||||
def _collect_sources(context: Path, dockerfile_path: Path) -> list[Path]:
|
||||
@dataclass(frozen=True)
|
||||
class StageInfo:
|
||||
name: str
|
||||
parent: str | None
|
||||
copies: tuple[CopyAdd, ...]
|
||||
|
||||
|
||||
def _parse_from_stage(line: str, fallback_index: int) -> tuple[str, str | None]:
|
||||
"""Return (stage_name, parent_stage_or_none)."""
|
||||
upper = line.upper()
|
||||
if not upper.startswith("FROM "):
|
||||
raise ValueError(line)
|
||||
rest = line[5:].strip()
|
||||
parts = rest.split()
|
||||
if len(parts) >= 3 and parts[-2].upper() == "AS":
|
||||
parent_token = " ".join(parts[:-2]).strip()
|
||||
name = parts[-1]
|
||||
elif len(parts) >= 1:
|
||||
parent_token = parts[0]
|
||||
name = f"stage{fallback_index}"
|
||||
else:
|
||||
name = f"stage{fallback_index}"
|
||||
parent_token = ""
|
||||
|
||||
parent: str | None
|
||||
if not parent_token or parent_token.lower() in (
|
||||
"scratch",
|
||||
) or "/" in parent_token or ":" in parent_token:
|
||||
# Image reference (python:3.12-slim) — not a stage alias.
|
||||
parent = None
|
||||
else:
|
||||
parent = parent_token
|
||||
return name, parent
|
||||
|
||||
|
||||
def _parse_stages(logical: list[str]) -> list[StageInfo]:
|
||||
stages: list[StageInfo] = []
|
||||
current_name: str | None = None
|
||||
current_parent: str | None = None
|
||||
current_copies: list[CopyAdd] = []
|
||||
fallback_index = 0
|
||||
|
||||
def flush() -> None:
|
||||
nonlocal current_name, current_parent, current_copies
|
||||
if current_name is None:
|
||||
return
|
||||
stages.append(
|
||||
StageInfo(
|
||||
name=current_name,
|
||||
parent=current_parent,
|
||||
copies=tuple(current_copies),
|
||||
)
|
||||
)
|
||||
current_copies = []
|
||||
|
||||
for ln in logical:
|
||||
upper = ln.lstrip().upper()
|
||||
if upper.startswith("FROM "):
|
||||
flush()
|
||||
current_name, current_parent = _parse_from_stage(ln, fallback_index)
|
||||
fallback_index += 1
|
||||
continue
|
||||
parsed = _parse_copy_add(ln)
|
||||
if parsed and not parsed.from_stage:
|
||||
current_copies.append(parsed)
|
||||
|
||||
flush()
|
||||
return stages
|
||||
|
||||
|
||||
def _stage_chain(stages: list[StageInfo], target: str | None) -> list[StageInfo]:
|
||||
if not stages:
|
||||
return []
|
||||
by_name = {s.name: s for s in stages}
|
||||
if not target:
|
||||
return list(stages)
|
||||
|
||||
if target not in by_name:
|
||||
return list(stages)
|
||||
|
||||
chain: list[StageInfo] = []
|
||||
seen: set[str] = set()
|
||||
name: str | None = target
|
||||
while name and name not in seen:
|
||||
seen.add(name)
|
||||
stage = by_name.get(name)
|
||||
if stage is None:
|
||||
break
|
||||
chain.append(stage)
|
||||
name = stage.parent
|
||||
chain.reverse()
|
||||
return chain
|
||||
|
||||
|
||||
def _collect_sources(context: Path, dockerfile_path: Path, *, target: str | None = None) -> list[Path]:
|
||||
text = _read_text(dockerfile_path)
|
||||
logical = _dockerfile_logical_lines(text)
|
||||
dockerignore_rules = _read_dockerignore_rules(context)
|
||||
stages = _parse_stages(logical)
|
||||
active_stages = _stage_chain(stages, target)
|
||||
|
||||
paths: list[Path] = []
|
||||
for ln in logical:
|
||||
parsed = _parse_copy_add(ln)
|
||||
if not parsed:
|
||||
continue
|
||||
if parsed.from_stage:
|
||||
continue
|
||||
copies: list[CopyAdd] = []
|
||||
if active_stages:
|
||||
for stage in active_stages:
|
||||
copies.extend(stage.copies)
|
||||
else:
|
||||
for ln in logical:
|
||||
parsed = _parse_copy_add(ln)
|
||||
if parsed and not parsed.from_stage:
|
||||
copies.append(parsed)
|
||||
|
||||
for parsed in copies:
|
||||
for src in parsed.sources:
|
||||
if _looks_like_remote(src):
|
||||
continue
|
||||
if src.startswith("/"):
|
||||
# Absolute COPY sources aren't valid for local context; ignore to avoid surprises.
|
||||
continue
|
||||
# Docker allows ".", "./foo", etc.
|
||||
src_norm = src.lstrip("./")
|
||||
if src_norm == "":
|
||||
src_norm = "."
|
||||
|
||||
if _is_glob(src_norm):
|
||||
# Expand within context
|
||||
for root, _, filenames in os.walk(context):
|
||||
root_p = Path(root)
|
||||
rel_root = root_p.relative_to(context).as_posix()
|
||||
@@ -290,7 +388,6 @@ def _collect_sources(context: Path, dockerfile_path: Path) -> list[Path]:
|
||||
continue
|
||||
|
||||
p = (context / src_norm).resolve()
|
||||
# Ensure stays within context
|
||||
try:
|
||||
p.relative_to(context.resolve())
|
||||
except Exception:
|
||||
@@ -304,13 +401,17 @@ def _collect_sources(context: Path, dockerfile_path: Path) -> list[Path]:
|
||||
continue
|
||||
paths.append(fp)
|
||||
|
||||
# Always include the Dockerfile itself (and preserve stable ordering via sort later)
|
||||
paths.append(dockerfile_path.resolve())
|
||||
return paths
|
||||
|
||||
|
||||
def compute_inputs_hash(context: Path, dockerfile_path: Path) -> str:
|
||||
files = _collect_sources(context=context, dockerfile_path=dockerfile_path)
|
||||
def compute_inputs_hash(
|
||||
context: Path,
|
||||
dockerfile_path: Path,
|
||||
*,
|
||||
target: str | None = None,
|
||||
) -> str:
|
||||
files = _collect_sources(context=context, dockerfile_path=dockerfile_path, target=target)
|
||||
# Deduplicate by resolved path
|
||||
uniq: dict[str, Path] = {}
|
||||
for p in files:
|
||||
@@ -344,11 +445,16 @@ def compute_inputs_hash(context: Path, dockerfile_path: Path) -> str:
|
||||
return h.hexdigest()
|
||||
|
||||
|
||||
def compute_inputs_debug(context: Path, dockerfile_path: Path) -> tuple[str, list[tuple[str, str]]]:
|
||||
def compute_inputs_debug(
|
||||
context: Path,
|
||||
dockerfile_path: Path,
|
||||
*,
|
||||
target: str | None = None,
|
||||
) -> tuple[str, list[tuple[str, str]]]:
|
||||
"""
|
||||
Returns (inputs_hash, [(rel_path, sha256_of_file_contents), ...]) with dockerignore applied.
|
||||
"""
|
||||
files = _collect_sources(context=context, dockerfile_path=dockerfile_path)
|
||||
files = _collect_sources(context=context, dockerfile_path=dockerfile_path, target=target)
|
||||
uniq: dict[str, Path] = {}
|
||||
for p in files:
|
||||
uniq[str(p.resolve())] = p.resolve()
|
||||
@@ -371,13 +477,18 @@ def compute_inputs_debug(context: Path, dockerfile_path: Path) -> tuple[str, lis
|
||||
else:
|
||||
items.append((rp, "NONFILE"))
|
||||
|
||||
return compute_inputs_hash(context=context, dockerfile_path=dockerfile_path), items
|
||||
return compute_inputs_hash(context=context, dockerfile_path=dockerfile_path, target=target), items
|
||||
|
||||
|
||||
def main() -> int:
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--context", required=True, help="Build context directory")
|
||||
ap.add_argument("--dockerfile", required=True, help="Dockerfile path")
|
||||
ap.add_argument(
|
||||
"--target",
|
||||
default="",
|
||||
help="Dockerfile build target (multi-stage); only COPY steps for that stage chain are hashed",
|
||||
)
|
||||
ap.add_argument(
|
||||
"--debug-list",
|
||||
action="store_true",
|
||||
@@ -387,6 +498,7 @@ def main() -> int:
|
||||
|
||||
context = Path(args.context).resolve()
|
||||
dockerfile = Path(args.dockerfile).resolve()
|
||||
target = args.target.strip() or None
|
||||
|
||||
if not context.exists() or not context.is_dir():
|
||||
print(f"Context not found or not a directory: {context}", file=sys.stderr)
|
||||
@@ -396,12 +508,12 @@ def main() -> int:
|
||||
return 2
|
||||
|
||||
if args.debug_list:
|
||||
h, items = compute_inputs_debug(context=context, dockerfile_path=dockerfile)
|
||||
h, items = compute_inputs_debug(context=context, dockerfile_path=dockerfile, target=target)
|
||||
for rp, sh in items:
|
||||
print(f"{rp}|{sh}", file=sys.stderr)
|
||||
print(h)
|
||||
else:
|
||||
print(compute_inputs_hash(context=context, dockerfile_path=dockerfile))
|
||||
print(compute_inputs_hash(context=context, dockerfile_path=dockerfile, target=target))
|
||||
return 0
|
||||
|
||||
|
||||
|
||||
@@ -172,11 +172,7 @@ def patch_updater_service(service: dict[str, Any], *, keep_build: bool) -> None:
|
||||
service.pop("volumes", None)
|
||||
return
|
||||
service["env_file"] = ["updater/.env", ".env"]
|
||||
environment = service.get("environment")
|
||||
if not isinstance(environment, dict):
|
||||
environment = {}
|
||||
environment["UPDATER_TOKEN"] = "${UPDATER_TOKEN}"
|
||||
service["environment"] = environment
|
||||
service.pop("environment", None)
|
||||
service["volumes"] = [
|
||||
"./compose.yml:/fromchat/compose.yml:rw",
|
||||
"/var/run/docker.sock:/var/run/docker.sock",
|
||||
|
||||
Reference in New Issue
Block a user