From 32e186213e31e3330bb6c0549ad68aef5477420c Mon Sep 17 00:00:00 2001 From: denis0001-dev Date: Wed, 1 Apr 2026 19:41:16 +0300 Subject: [PATCH] Optimize deployment script --- .dockerignore | 44 +++ .gitignore | 4 +- scripts/deploy.sh | 672 ++++++++++------------------------ scripts/docker_inputs_hash.py | 369 +++++++++++++++++++ 4 files changed, 617 insertions(+), 472 deletions(-) create mode 100644 scripts/docker_inputs_hash.py diff --git a/.dockerignore b/.dockerignore index 68fb1c2..f48cdff 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,2 +1,46 @@ +# Shared build context ignore (used by deployment images with context: ..) + +# VCS / editor +.git +.github +.idea +.vscode +.DS_Store + +# Secrets / local env +.env +deployment/.env + +# Node +node_modules +npm-debug.log +.vite +dist +dist-electron +build +out +coverage + +# Python +__pycache__/ +*.pyc +.pytest_cache/ +.mypy_cache/ +.ipynb_checkpoints +.venv +venv/ + +# App runtime data/logs (mounted, not baked) +backend/data +backend/files +data +logs +**/logs +**/logs/** +*.log + +# Deploy cache (hashes) +.deploy-cache + # Firebase cert: bind-mounted at runtime; do not send to docker build context firebase-cert.json diff --git a/.gitignore b/.gitignore index e21a645..61b606c 100644 --- a/.gitignore +++ b/.gitignore @@ -591,5 +591,7 @@ backend/files *.db-wal *.db-shm +.deploy-cache + !scripts/offline_python_linux_amd64/venv_linux_amd64/venv/ -!scripts/offline_python_linux_amd64/venv_linux_amd64/venv/lib \ No newline at end of file +!scripts/offline_python_linux_amd64/venv_linux_amd64/venv/lib diff --git a/scripts/deploy.sh b/scripts/deploy.sh index e032e6c..6ac2dab 100755 --- a/scripts/deploy.sh +++ b/scripts/deploy.sh @@ -1,5 +1,5 @@ #!/bin/bash -set -e +set -euo pipefail # Complete deployment script: build and push to server # Usage: ./scripts/deploy.sh [server_user@server_host] [deployment_path] [platform] @@ -22,7 +22,7 @@ warning() { echo -e "${YELLOW}⚠${NC} $1"; } error() { echo -e "${RED}✗${NC} $1"; } step() { echo -e "${CYAN}${BOLD}→${NC} ${BOLD}$1${NC}"; } substep() { - if [ "$2" = "-n" ]; then + if [ "${2:-}" = "-n" ]; then echo -n -e " ${GREEN}•${NC} $1" else echo -e " ${GREEN}•${NC} $1" @@ -64,6 +64,12 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" DEPLOYMENT_DIR="$PROJECT_ROOT/deployment" ENV_FILE="$DEPLOYMENT_DIR/.env" +INPUT_HASH_TOOL="$SCRIPT_DIR/docker_inputs_hash.py" +LOCAL_CACHE_ROOT="$PROJECT_ROOT/.deploy-cache" +LOCAL_IMAGE_CACHE_DIR="$LOCAL_CACHE_ROOT/images" + +# Ensure arrays exist even when build is skipped (nounset-safe). +declare -a BUILT_IMAGES # Load .env file if it exists if [ -f "$ENV_FILE" ]; then @@ -102,6 +108,37 @@ if [ -n "$HOST_ARCH" ] && [ "$HOST_ARCH" = "$PLATFORM_ARCH" ]; then USE_DOCKER_BUILD=true fi +sanitize_ref() { + # Replace characters that are problematic for file paths. + # Example: "foo/bar:baz" -> "foo_bar__baz" + local s="$1" + s="${s//\//_}" + s="${s//:/__}" + s="${s//@/__at__}" + echo "$s" +} + +read_file_if_exists() { + local p="$1" + if [ -f "$p" ]; then + cat "$p" + fi +} + +# Docker's top-level image Id can differ for the same layers (OCI index vs image, BuildKit attestations). +# Compare a fingerprint of the final layer stack instead. +get_local_image_layer_fp() { + local img="$1" + docker image inspect -f '{{json .RootFS.Layers}}' "$img" 2>/dev/null \ + | openssl dgst -sha256 2>/dev/null | awk '{print $2}' || true +} + +compute_inputs_hash() { + local context="$1" + local dockerfile="$2" + python3 "$INPUT_HASH_TOOL" --context "$context" --dockerfile "$dockerfile" +} + # Check if server is provided if [ -z "$SERVER" ]; then error "Server not specified. Usage: $0 [user@host] [deployment_path] [platform]" @@ -299,6 +336,12 @@ if ! COMPOSE_JSON=$(docker compose -f docker-compose.yml config --format json 2> fi BUILT_IMAGES=() +PUSHABLE_IMAGES=() +PUSHABLE_SERVICES=() +PUSHABLE_SERVICE_DOCKERFILES=() +PUSHABLE_SERVICE_CONTEXTS=() +PUSHABLE_SERVICE_TARGETS=() +PUSHABLE_IMAGE_INPUT_HASHES=() for SERVICE in $SERVICES; do if ! jq -e --arg s "$SERVICE" '(.services[$s].build // false) | type == "object"' <<< "$COMPOSE_JSON" >/dev/null 2>&1; then @@ -307,8 +350,6 @@ for SERVICE in $SERVICES; do IMAGE_TAG="${PROJECT_NAME}-${SERVICE}:latest" - substep "Building ${CYAN}$SERVICE${NC} -> ${CYAN}$IMAGE_TAG${NC}..." - DOCKERFILE_REL=$(jq -r --arg s "$SERVICE" '.services[$s].build.dockerfile // empty' <<< "$COMPOSE_JSON") CONTEXT_REL=$(jq -r --arg s "$SERVICE" '.services[$s].build.context // empty' <<< "$COMPOSE_JSON") # Multi-stage deployment/Dockerfile: without --target, the final stage (file_storage) is always tagged. @@ -345,443 +386,101 @@ for SERVICE in $SERVICES; do error "Could not determine Dockerfile for $SERVICE" fi fi - - if [ "$USE_DOCKER_BUILD" = true ]; then - DOCKER_BUILD_ARGS=(build --platform "$PLATFORM" --file "$DOCKERFILE" --tag "$IMAGE_TAG") - if [ -n "$BUILD_TARGET" ]; then - DOCKER_BUILD_ARGS+=(--target "$BUILD_TARGET") - fi - DOCKER_BUILD_ARGS+=("$BUILD_CONTEXT") - if docker "${DOCKER_BUILD_ARGS[@]}"; then - echo -e " ${GREEN}✓${NC} Built ${CYAN}$SERVICE${NC}" - BUILT_IMAGES+=("$IMAGE_TAG") - echo "" - else - error "Build failed for $SERVICE" - exit 1 - fi - else - # On macOS Docker Desktop, --load can hang for a long time at "sending tarball". - # Use the docker exporter explicitly to load into the local Docker daemon. - BUILDX_ARGS=(buildx build --platform "$PLATFORM" --file "$DOCKERFILE" --tag "$IMAGE_TAG" --output=type=docker) - if [ -n "$BUILD_TARGET" ]; then - BUILDX_ARGS+=(--target "$BUILD_TARGET") - fi - BUILDX_ARGS+=("$BUILD_CONTEXT") - - if docker "${BUILDX_ARGS[@]}"; then - echo -e " ${GREEN}✓${NC} Built ${CYAN}$SERVICE${NC}" - BUILT_IMAGES+=("$IMAGE_TAG") - echo "" - else - error "Build failed for $SERVICE" - exit 1 - fi - fi - - true - - if [ "$USE_DOCKER_BUILD" = false ]; then - true - fi - - if [ "$USE_DOCKER_BUILD" = true ]; then - true - fi - - if [ "$USE_DOCKER_BUILD" = false ]; then - true - fi - - if [ "$USE_DOCKER_BUILD" = true ]; then - true - fi - - if [ "$USE_DOCKER_BUILD" = false ]; then - true - fi - - if [ "$USE_DOCKER_BUILD" = true ]; then - true - fi - - if [ "$USE_DOCKER_BUILD" = false ]; then - true - fi - - if [ "$USE_DOCKER_BUILD" = true ]; then - true - fi - - if [ "$USE_DOCKER_BUILD" = false ]; then - true - fi - - if [ "$USE_DOCKER_BUILD" = true ]; then - true - fi - - if [ "$USE_DOCKER_BUILD" = false ]; then - true - fi - - if [ "$USE_DOCKER_BUILD" = true ]; then - true - fi - - if [ "$USE_DOCKER_BUILD" = false ]; then - true - fi - - if [ "$USE_DOCKER_BUILD" = true ]; then - true - fi - - if [ "$USE_DOCKER_BUILD" = false ]; then - true - fi - - if [ "$USE_DOCKER_BUILD" = true ]; then - true - fi - - if [ "$USE_DOCKER_BUILD" = false ]; then - true - fi - - if [ "$USE_DOCKER_BUILD" = true ]; then - true - fi - - if [ "$USE_DOCKER_BUILD" = false ]; then - true - fi - - if [ "$USE_DOCKER_BUILD" = true ]; then - true - fi - - if [ "$USE_DOCKER_BUILD" = false ]; then - true - fi - - if [ "$USE_DOCKER_BUILD" = true ]; then - true - fi - - if [ "$USE_DOCKER_BUILD" = false ]; then - true - fi - - if [ "$USE_DOCKER_BUILD" = true ]; then - true - fi - - if [ "$USE_DOCKER_BUILD" = false ]; then - true - fi - - if [ "$USE_DOCKER_BUILD" = true ]; then - true - fi - - if [ "$USE_DOCKER_BUILD" = false ]; then - true - fi - - if [ "$USE_DOCKER_BUILD" = true ]; then - true - fi - - if [ "$USE_DOCKER_BUILD" = false ]; then - true - fi - - if [ "$USE_DOCKER_BUILD" = true ]; then - true - fi - - if [ "$USE_DOCKER_BUILD" = false ]; then - true - fi - - if [ "$USE_DOCKER_BUILD" = true ]; then - true - fi - - if [ "$USE_DOCKER_BUILD" = false ]; then - true - fi - - if [ "$USE_DOCKER_BUILD" = true ]; then - true - fi - - if [ "$USE_DOCKER_BUILD" = false ]; then - true - fi - - if [ "$USE_DOCKER_BUILD" = true ]; then - true - fi - - if [ "$USE_DOCKER_BUILD" = false ]; then - true - fi - - if [ "$USE_DOCKER_BUILD" = true ]; then - true - fi - - if [ "$USE_DOCKER_BUILD" = false ]; then - true - fi - - if [ "$USE_DOCKER_BUILD" = true ]; then - true - fi - - if [ "$USE_DOCKER_BUILD" = false ]; then - true - fi - - if [ "$USE_DOCKER_BUILD" = true ]; then - true - fi - - if [ "$USE_DOCKER_BUILD" = false ]; then - true - fi - - if [ "$USE_DOCKER_BUILD" = true ]; then - true - fi - - if [ "$USE_DOCKER_BUILD" = false ]; then - true - fi - - if [ "$USE_DOCKER_BUILD" = true ]; then - true - fi - - if [ "$USE_DOCKER_BUILD" = false ]; then - true - fi - - if [ "$USE_DOCKER_BUILD" = true ]; then - true - fi - - if [ "$USE_DOCKER_BUILD" = false ]; then - true + if [ ! -f "$INPUT_HASH_TOOL" ]; then + error "Missing $INPUT_HASH_TOOL (needed for dependency hashing)" + exit 1 fi - if [ "$USE_DOCKER_BUILD" = true ]; then - true - fi - - if [ "$USE_DOCKER_BUILD" = false ]; then - true - fi - - if [ "$USE_DOCKER_BUILD" = true ]; then - true - fi - - if [ "$USE_DOCKER_BUILD" = false ]; then - true - fi - - if [ "$USE_DOCKER_BUILD" = true ]; then - true - fi - - if [ "$USE_DOCKER_BUILD" = false ]; then - true - fi - - if [ "$USE_DOCKER_BUILD" = true ]; then - true - fi - - if [ "$USE_DOCKER_BUILD" = false ]; then - true - fi - - if [ "$USE_DOCKER_BUILD" = true ]; then - true - fi - - if [ "$USE_DOCKER_BUILD" = false ]; then - true - fi - - if [ "$USE_DOCKER_BUILD" = true ]; then - true - fi - - if [ "$USE_DOCKER_BUILD" = false ]; then - true - fi - - if [ "$USE_DOCKER_BUILD" = true ]; then - true - fi - - if [ "$USE_DOCKER_BUILD" = false ]; then - true - fi - - if [ "$USE_DOCKER_BUILD" = true ]; then - true - fi - - if [ "$USE_DOCKER_BUILD" = false ]; then - true - fi - - if [ "$USE_DOCKER_BUILD" = true ]; then - true - fi - - if [ "$USE_DOCKER_BUILD" = false ]; then - true - fi - - if [ "$USE_DOCKER_BUILD" = true ]; then - true - fi - - if [ "$USE_DOCKER_BUILD" = false ]; then - true - fi - - if [ "$USE_DOCKER_BUILD" = true ]; then - true - fi - - if [ "$USE_DOCKER_BUILD" = false ]; then - true - fi - - if [ "$USE_DOCKER_BUILD" = true ]; then - true - fi - - if [ "$USE_DOCKER_BUILD" = false ]; then - true - fi - - if [ "$USE_DOCKER_BUILD" = true ]; then - true - fi - - if [ "$USE_DOCKER_BUILD" = false ]; then - true - fi - - if [ "$USE_DOCKER_BUILD" = true ]; then - true - fi - - if [ "$USE_DOCKER_BUILD" = false ]; then - true - fi - - if [ "$USE_DOCKER_BUILD" = true ]; then - true - fi - - if [ "$USE_DOCKER_BUILD" = false ]; then - true - fi - - if [ "$USE_DOCKER_BUILD" = true ]; then - true - fi - - if [ "$USE_DOCKER_BUILD" = false ]; then - true - fi - - if [ "$USE_DOCKER_BUILD" = true ]; then - true - fi - - if [ "$USE_DOCKER_BUILD" = false ]; then - true - fi - - if [ "$USE_DOCKER_BUILD" = true ]; then - true - fi - - if [ "$USE_DOCKER_BUILD" = false ]; then - true - fi - - if [ "$USE_DOCKER_BUILD" = true ]; then - true - fi - - if [ "$USE_DOCKER_BUILD" = false ]; then - true - fi - - if [ "$USE_DOCKER_BUILD" = true ]; then - true - fi - - if [ "$USE_DOCKER_BUILD" = false ]; then - true - fi - - if [ "$USE_DOCKER_BUILD" = true ]; then - true - fi - - if [ "$USE_DOCKER_BUILD" = false ]; then - true - fi - - if [ "$USE_DOCKER_BUILD" = true ]; then - true - fi - - if [ "$USE_DOCKER_BUILD" = false ]; then - true - fi - - if [ "$USE_DOCKER_BUILD" = true ]; then - true - fi - - if [ "$USE_DOCKER_BUILD" = false ]; then - true - fi - - if [ "$USE_DOCKER_BUILD" = true ]; then - true - fi - - if [ "$USE_DOCKER_BUILD" = false ]; then - true - fi - - if [ "$USE_DOCKER_BUILD" = true ]; then - true - fi - - if [ "$USE_DOCKER_BUILD" = false ]; then - true - fi + INPUT_HASH=$(compute_inputs_hash "$BUILD_CONTEXT" "$DOCKERFILE") - true + PUSHABLE_SERVICES+=("$SERVICE") + PUSHABLE_IMAGES+=("$IMAGE_TAG") + PUSHABLE_SERVICE_DOCKERFILES+=("$DOCKERFILE") + PUSHABLE_SERVICE_CONTEXTS+=("$BUILD_CONTEXT") + PUSHABLE_SERVICE_TARGETS+=("$BUILD_TARGET") + PUSHABLE_IMAGE_INPUT_HASHES+=("$INPUT_HASH") done -success "Build complete! ${#BUILT_IMAGES[@]} image(s) ready" +mkdir -p "$LOCAL_IMAGE_CACHE_DIR" + +SERVICES_TO_BUILD=() +IMAGES_TO_BUILD=() +INPUT_HASHES_TO_BUILD=() +DOCKERFILES_TO_BUILD=() +CONTEXTS_TO_BUILD=() +TARGETS_TO_BUILD=() + +for i in "${!PUSHABLE_SERVICES[@]}"; do + SERVICE="${PUSHABLE_SERVICES[$i]}" + IMAGE_TAG="${PUSHABLE_IMAGES[$i]}" + INPUT_HASH="${PUSHABLE_IMAGE_INPUT_HASHES[$i]}" + DOCKERFILE="${PUSHABLE_SERVICE_DOCKERFILES[$i]}" + BUILD_CONTEXT="${PUSHABLE_SERVICE_CONTEXTS[$i]}" + BUILD_TARGET="${PUSHABLE_SERVICE_TARGETS[$i]}" + + CACHE_KEY="$(sanitize_ref "$IMAGE_TAG")" + CACHE_FILE="$LOCAL_IMAGE_CACHE_DIR/$CACHE_KEY/input.sha256" + PREV_HASH="$(read_file_if_exists "$CACHE_FILE")" + LOCAL_FP="$(get_local_image_layer_fp "$IMAGE_TAG")" + + if [ -n "$PREV_HASH" ] && [ "$PREV_HASH" = "$INPUT_HASH" ] && [ -n "$LOCAL_FP" ]; then + continue + fi + + SERVICES_TO_BUILD+=("$SERVICE") + IMAGES_TO_BUILD+=("$IMAGE_TAG") + INPUT_HASHES_TO_BUILD+=("$INPUT_HASH") + DOCKERFILES_TO_BUILD+=("$DOCKERFILE") + CONTEXTS_TO_BUILD+=("$BUILD_CONTEXT") + TARGETS_TO_BUILD+=("$BUILD_TARGET") +done + +if [ ${#SERVICES_TO_BUILD[@]} -eq 0 ]; then + success "Build skipped (no Docker inputs changed)" +else + step "Building ${#SERVICES_TO_BUILD[@]} service(s)" + + if [ "$USE_DOCKER_BUILD" = true ]; then + # Same-arch fast path: build with Docker Compose (requested). + # Ensure the project name is stable so the resulting tags match our expectations. + COMPOSE_PROJECT_NAME="$PROJECT_NAME" docker compose -f docker-compose.yml --profile production build "${SERVICES_TO_BUILD[@]}" + else + for i in "${!SERVICES_TO_BUILD[@]}"; do + SERVICE="${SERVICES_TO_BUILD[$i]}" + IMAGE_TAG="${IMAGES_TO_BUILD[$i]}" + DOCKERFILE="${DOCKERFILES_TO_BUILD[$i]}" + BUILD_CONTEXT="${CONTEXTS_TO_BUILD[$i]}" + BUILD_TARGET="${TARGETS_TO_BUILD[$i]}" + + substep "Building ${CYAN}$SERVICE${NC} -> ${CYAN}$IMAGE_TAG${NC}..." + + # On macOS Docker Desktop, --load can hang for a long time at "sending tarball". + # Use the docker exporter explicitly to load into the local Docker daemon. + BUILDX_ARGS=(buildx build --platform "$PLATFORM" --file "$DOCKERFILE" --tag "$IMAGE_TAG" --output=type=docker --provenance=false --sbom=false) + if [ -n "$BUILD_TARGET" ]; then + BUILDX_ARGS+=(--target "$BUILD_TARGET") + fi + BUILDX_ARGS+=("$BUILD_CONTEXT") + + if ! docker "${BUILDX_ARGS[@]}"; then + error "Build failed for $SERVICE" + exit 1 + fi + done + fi + + for i in "${!SERVICES_TO_BUILD[@]}"; do + IMAGE_TAG="${IMAGES_TO_BUILD[$i]}" + INPUT_HASH="${INPUT_HASHES_TO_BUILD[$i]}" + CACHE_KEY="$(sanitize_ref "$IMAGE_TAG")" + mkdir -p "$LOCAL_IMAGE_CACHE_DIR/$CACHE_KEY" + printf "%s" "$INPUT_HASH" > "$LOCAL_IMAGE_CACHE_DIR/$CACHE_KEY/input.sha256" + BUILT_IMAGES+=("$IMAGE_TAG") + done + + success "Build complete! ${#BUILT_IMAGES[@]} image(s) built" +fi # ============================================================================ # DEPLOY PHASE @@ -837,31 +536,36 @@ fi BUILT_COUNT=${#BUILT_IMAGES[@]} MATCHING_BUILT=0 MISSING_FROM_DETECTED=() -for BI in "${BUILT_IMAGES[@]}"; do - found=false - for DI in "${PUSH_IMAGES[@]}"; do - if [ "$BI" = "$DI" ]; then - found=true - break +# Bash 3.2 + set -u: "${arr[@]}" on an empty array can error; guard with length. +if ((${#BUILT_IMAGES[@]} > 0)); then + for BI in "${BUILT_IMAGES[@]}"; do + found=false + for DI in "${PUSH_IMAGES[@]}"; do + if [ "$BI" = "$DI" ]; then + found=true + break + fi + done + if [ "$found" = true ]; then + MATCHING_BUILT=$((MATCHING_BUILT + 1)) + else + MISSING_FROM_DETECTED+=("$BI") fi done - if [ "$found" = true ]; then - MATCHING_BUILT=$((MATCHING_BUILT + 1)) - else - MISSING_FROM_DETECTED+=("$BI") - fi -done +fi # Also list push images that weren't built locally (these are likely prebuilt local images) NOT_BUILT_DETECTED=() for DI in "${PUSH_IMAGES[@]}"; do built=false - for BI in "${BUILT_IMAGES[@]}"; do - if [ "$DI" = "$BI" ]; then - built=true - break - fi - done + if ((${#BUILT_IMAGES[@]} > 0)); then + for BI in "${BUILT_IMAGES[@]}"; do + if [ "$DI" = "$BI" ]; then + built=true + break + fi + done + fi if [ "$built" = false ]; then NOT_BUILT_DETECTED+=("$DI") fi @@ -883,33 +587,59 @@ if [ ${#PUSH_IMAGES[@]} -eq 0 ] && [ ${#EXTERNAL_IMAGES[@]} -eq 0 ]; then error "No images found in docker-compose.yml or built locally for project ${PROJECT_NAME}" fi -# Pre-pull unregistry image if needed -UNREGISTRY_IMAGE="ghcr.io/psviderski/unregistry" -if ! ssh "$SERVER" "sudo docker images --format '{{.Repository}}:{{.Tag}}' | grep -q '^${UNREGISTRY_IMAGE}$'" 2>/dev/null; then - substep "Pulling unregistry image (one-time setup)..." - ssh -tt "$SERVER" "sudo docker pull ${UNREGISTRY_IMAGE}" +# Transfer images — only what was rebuilt this run (BUILT_IMAGES). No remote hash/fingerprint checks. +step "Transferring images" + +IMAGES_TO_PUSH=() +if ((${#BUILT_IMAGES[@]} > 0)); then + for PI in "${PUSH_IMAGES[@]}"; do + for BI in "${BUILT_IMAGES[@]}"; do + if [ "$PI" = "$BI" ]; then + IMAGES_TO_PUSH+=("$PI") + break + fi + done + done fi -# Transfer images -step "Transferring images" -for IMAGE in "${PUSH_IMAGES[@]}"; do - substep "Pushing ${CYAN}$IMAGE${NC}..." - if docker pussh "$IMAGE" "$SERVER"; then - echo "" - else - echo -e " ${RED}✗${NC} Failed to push ${CYAN}$IMAGE${NC}" - echo "" - exit 1 +if [ ${#IMAGES_TO_PUSH[@]} -eq 0 ]; then + success "Skipping image push (nothing was rebuilt this run)" +else + UNREGISTRY_IMAGE="ghcr.io/psviderski/unregistry" + if ! ssh "$SERVER" "sudo docker images --format '{{.Repository}}:{{.Tag}}' | grep -q '^${UNREGISTRY_IMAGE}$'" 2>/dev/null; then + substep "Pulling unregistry image (one-time setup)..." + if [ -n "${SUDO_PASSWORD:-}" ]; then + ssh "$SERVER" "echo '$SUDO_PASSWORD' | sudo -S -p '' docker pull ${UNREGISTRY_IMAGE}" + else + ssh "$SERVER" "sudo docker pull ${UNREGISTRY_IMAGE}" + fi fi -done + for IMAGE in "${IMAGES_TO_PUSH[@]}"; do + substep "Pushing ${CYAN}$IMAGE${NC}..." + if ! docker pussh "$IMAGE" "$SERVER"; then + echo -e " ${RED}✗${NC} Failed to push ${CYAN}$IMAGE${NC}" + echo "" + exit 1 + fi + echo "" + done +fi # Pull external images directly on the server (no pussh) if [ ${#EXTERNAL_IMAGES[@]} -gt 0 ]; then step "Pulling external images on server" for IMAGE in "${EXTERNAL_IMAGES[@]}"; do substep "Pulling ${CYAN}$IMAGE${NC}..." - # Allocate a TTY and do not redirect output so failures are visible. - if ssh -tt "$SERVER" "sudo docker pull $(printf '%q' "$IMAGE")"; then + QIMG=$(printf '%q' "$IMAGE") + if [ -n "${SUDO_PASSWORD:-}" ]; then + if ssh "$SERVER" "echo '$SUDO_PASSWORD' | sudo -S -p '' docker pull $QIMG"; then + echo "" + else + echo -e " ${RED}✗${NC} Failed to pull ${CYAN}$IMAGE${NC} on server" + echo "" + exit 1 + fi + elif ssh "$SERVER" "sudo docker pull $QIMG"; then echo "" else echo -e " ${RED}✗${NC} Failed to pull ${CYAN}$IMAGE${NC} on server" diff --git a/scripts/docker_inputs_hash.py b/scripts/docker_inputs_hash.py new file mode 100644 index 0000000..c1add15 --- /dev/null +++ b/scripts/docker_inputs_hash.py @@ -0,0 +1,369 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import fnmatch +import hashlib +import json +import os +import shlex +import sys +from dataclasses import dataclass +from pathlib import Path + + +def _sha256_bytes(data: bytes) -> str: + h = hashlib.sha256() + h.update(data) + return h.hexdigest() + + +def _sha256_file(path: Path) -> str: + h = hashlib.sha256() + with path.open("rb") as f: + for chunk in iter(lambda: f.read(1024 * 1024), b""): + h.update(chunk) + return h.hexdigest() + + +def _read_text(path: Path) -> str: + return path.read_text(encoding="utf-8", errors="replace") + + +@dataclass(frozen=True) +class DockerIgnoreRule: + pattern: str + negated: bool + anchored: bool + directory_only: bool + + +def _read_dockerignore_rules(context: Path) -> list[DockerIgnoreRule]: + p = context / ".dockerignore" + if not p.exists() or not p.is_file(): + return [] + + rules: list[DockerIgnoreRule] = [] + for raw in _read_text(p).splitlines(): + line = raw.strip() + if not line or line.startswith("#"): + continue + negated = line.startswith("!") + if negated: + line = line[1:].lstrip() + anchored = line.startswith("/") + if anchored: + line = line[1:] + directory_only = line.endswith("/") + if directory_only: + line = line[:-1] + if not line: + continue + rules.append( + DockerIgnoreRule( + pattern=line, + negated=negated, + anchored=anchored, + directory_only=directory_only, + ) + ) + return rules + + +def _dockerignore_matches(rule: DockerIgnoreRule, rel_posix: str, is_dir: bool) -> bool: + if rule.directory_only and not is_dir: + return False + + rel = rel_posix.lstrip("./") + if rule.anchored: + # Anchored to context root. + candidates = [rel] + else: + # Unanchored patterns match anywhere: try both full rel and basename. + base = rel.rsplit("/", 1)[-1] + candidates = [rel, base] + + # Dockerignore supports ** globs; fnmatch handles this well enough for our use. + for c in candidates: + if fnmatch.fnmatch(c, rule.pattern): + return True + # Also allow matching directory prefixes for patterns like "dist" against "foo/dist/bar". + if not rule.anchored and "/" in rel: + if fnmatch.fnmatch(rel, f"*/{rule.pattern}") or fnmatch.fnmatch(rel, f"**/{rule.pattern}"): + return True + return False + + +def _is_ignored_by_dockerignore(rules: list[DockerIgnoreRule], rel_posix: str, is_dir: bool) -> bool: + ignored = False + for r in rules: + if _dockerignore_matches(r, rel_posix=rel_posix, is_dir=is_dir): + ignored = not r.negated + return ignored + + +def _dockerfile_logical_lines(dockerfile_text: str) -> list[str]: + """ + Join backslash-continued lines and drop full-line comments. + """ + out: list[str] = [] + buf: list[str] = [] + for raw in dockerfile_text.splitlines(): + line = raw.rstrip() + if not buf: + stripped = line.lstrip() + if stripped.startswith("#") or stripped == "": + continue + buf.append(line) + if line.endswith("\\"): + buf[-1] = buf[-1][:-1].rstrip() + continue + joined = " ".join(x.strip() for x in buf if x.strip()) + buf = [] + if joined: + out.append(joined) + if buf: + joined = " ".join(x.strip() for x in buf if x.strip()) + if joined: + out.append(joined) + return out + + +@dataclass(frozen=True) +class CopyAdd: + sources: tuple[str, ...] + from_stage: bool + + +def _parse_copy_add_args_shellform(args: list[str]) -> CopyAdd | None: + # flags: --from=, --chown=, --chmod=, --link, --parents, --exclude=... etc. + from_stage = False + rest: list[str] = [] + for a in args: + if a.startswith("--from=") or a == "--from": + from_stage = True + continue + if a.startswith("--"): + continue + rest.append(a) + if len(rest) < 2: + return None + # last is dest + srcs = tuple(rest[:-1]) + return CopyAdd(sources=srcs, from_stage=from_stage) + + +def _parse_copy_add_args_jsonform(json_text: str) -> CopyAdd | None: + try: + arr = json.loads(json_text) + except Exception: + return None + if not isinstance(arr, list) or len(arr) < 2: + return None + # last is dest + srcs = tuple(x for x in arr[:-1] if isinstance(x, str)) + if not srcs: + return None + return CopyAdd(sources=srcs, from_stage=False) + + +def _parse_copy_add(line: str) -> CopyAdd | None: + upper = line.lstrip().upper() + if not (upper.startswith("COPY ") or upper.startswith("ADD ")): + return None + + # Keep original casing for paths. + keyword, rest = line.split(None, 1) + rest = rest.strip() + + # JSON form starts with '[' + if rest.startswith("["): + parsed = _parse_copy_add_args_jsonform(rest) + if parsed: + return parsed + return None + + # shell form + try: + parts = shlex.split(rest, posix=True) + except Exception: + return None + return _parse_copy_add_args_shellform(parts) + + +def _looks_like_remote(src: str) -> bool: + s = src.lower() + return s.startswith("http://") or s.startswith("https://") + + +def _is_glob(p: str) -> bool: + return any(ch in p for ch in ["*", "?", "["]) + + +def _iter_files_under(path: Path) -> list[Path]: + if not path.exists(): + return [] + if path.is_file(): + return [path] + files: list[Path] = [] + for root, _, filenames in os.walk(path): + for name in filenames: + files.append(Path(root) / name) + return files + + +def _collect_sources(context: Path, dockerfile_path: Path) -> list[Path]: + text = _read_text(dockerfile_path) + logical = _dockerfile_logical_lines(text) + dockerignore_rules = _read_dockerignore_rules(context) + + paths: list[Path] = [] + for ln in logical: + parsed = _parse_copy_add(ln) + if not parsed: + continue + if parsed.from_stage: + continue + for src in parsed.sources: + if _looks_like_remote(src): + continue + if src.startswith("/"): + # Absolute COPY sources aren't valid for local context; ignore to avoid surprises. + continue + # Docker allows ".", "./foo", etc. + src_norm = src.lstrip("./") + if src_norm == "": + src_norm = "." + + if _is_glob(src_norm): + # Expand within context + for root, _, filenames in os.walk(context): + root_p = Path(root) + rel_root = root_p.relative_to(context).as_posix() + for fn in filenames: + rel = f"{rel_root}/{fn}" if rel_root != "." else fn + if _is_ignored_by_dockerignore(dockerignore_rules, rel_posix=rel, is_dir=False): + continue + if fnmatch.fnmatch(rel, src_norm) or fnmatch.fnmatch(fn, src_norm): + paths.append(context / rel) + continue + + p = (context / src_norm).resolve() + # Ensure stays within context + try: + p.relative_to(context.resolve()) + except Exception: + continue + for fp in _iter_files_under(p): + try: + rel = fp.resolve().relative_to(context.resolve()).as_posix() + except Exception: + continue + if _is_ignored_by_dockerignore(dockerignore_rules, rel_posix=rel, is_dir=fp.is_dir()): + continue + paths.append(fp) + + # Always include the Dockerfile itself (and preserve stable ordering via sort later) + paths.append(dockerfile_path.resolve()) + return paths + + +def compute_inputs_hash(context: Path, dockerfile_path: Path) -> str: + files = _collect_sources(context=context, dockerfile_path=dockerfile_path) + # Deduplicate by resolved path + uniq: dict[str, Path] = {} + for p in files: + uniq[str(p)] = p + + # Stable sort by path relative to context when possible, else absolute + ctx_resolved = context.resolve() + def sort_key(p: Path) -> str: + try: + return p.resolve().relative_to(ctx_resolved).as_posix() + except Exception: + return p.resolve().as_posix() + + sorted_files = sorted(uniq.values(), key=sort_key) + + h = hashlib.sha256() + for p in sorted_files: + rp: str + try: + rp = p.resolve().relative_to(ctx_resolved).as_posix() + except Exception: + rp = p.resolve().as_posix() + h.update(rp.encode("utf-8", errors="strict")) + h.update(b"\0") + if p.is_file(): + h.update(_sha256_file(p).encode("ascii")) + else: + h.update(b"NONFILE") + h.update(b"\n") + + return h.hexdigest() + + +def compute_inputs_debug(context: Path, dockerfile_path: Path) -> tuple[str, list[tuple[str, str]]]: + """ + Returns (inputs_hash, [(rel_path, sha256_of_file_contents), ...]) with dockerignore applied. + """ + files = _collect_sources(context=context, dockerfile_path=dockerfile_path) + uniq: dict[str, Path] = {} + for p in files: + uniq[str(p.resolve())] = p.resolve() + + ctx_resolved = context.resolve() + + def rel_or_abs(p: Path) -> str: + try: + return p.resolve().relative_to(ctx_resolved).as_posix() + except Exception: + return p.resolve().as_posix() + + sorted_files = sorted(uniq.values(), key=lambda p: rel_or_abs(p)) + + items: list[tuple[str, str]] = [] + for p in sorted_files: + rp = rel_or_abs(p) + if p.is_file(): + items.append((rp, _sha256_file(p))) + else: + items.append((rp, "NONFILE")) + + return compute_inputs_hash(context=context, dockerfile_path=dockerfile_path), items + + +def main() -> int: + ap = argparse.ArgumentParser() + ap.add_argument("--context", required=True, help="Build context directory") + ap.add_argument("--dockerfile", required=True, help="Dockerfile path") + ap.add_argument( + "--debug-list", + action="store_true", + help="Print the included file list (relpath|sha256) to stderr", + ) + args = ap.parse_args() + + context = Path(args.context).resolve() + dockerfile = Path(args.dockerfile).resolve() + + if not context.exists() or not context.is_dir(): + print(f"Context not found or not a directory: {context}", file=sys.stderr) + return 2 + if not dockerfile.exists() or not dockerfile.is_file(): + print(f"Dockerfile not found: {dockerfile}", file=sys.stderr) + return 2 + + if args.debug_list: + h, items = compute_inputs_debug(context=context, dockerfile_path=dockerfile) + for rp, sh in items: + print(f"{rp}|{sh}", file=sys.stderr) + print(h) + else: + print(compute_inputs_hash(context=context, dockerfile_path=dockerfile)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) +