FromChat/web
1
0
Fork 0
mirror of https://github.com/fromchat-messenger/web.git synced 2026-09-22 19:15:08 +03:00
Code Issues Packages Projects Releases Wiki Activity

Optimize deployment script

Browse Source
This commit is contained in:
denis0001-dev
2026-04-01 19:41:16 +03:00
Unverified
parent 07871bbde9
commit 32e186213e
4 changed files with 617 additions and 472 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.dockerignore
+44
View File
@@ -1,2 +1,46 @@
# Shared build context ignore (used by deployment images with context: ..)
# VCS / editor
.git
.github
.idea
.vscode
.DS_Store
# Secrets / local env
.env
deployment/.env
# Node
node_modules
npm-debug.log
.vite
dist
dist-electron
build
out
coverage
# Python
__pycache__/
*.pyc
.pytest_cache/
.mypy_cache/
.ipynb_checkpoints
.venv
venv/
# App runtime data/logs (mounted, not baked)
backend/data
backend/files
data
logs
**/logs
**/logs/**
*.log
# Deploy cache (hashes)
.deploy-cache
# Firebase cert: bind-mounted at runtime; do not send to docker build context
firebase-cert.json
.gitignore
+2
View File
@@ -591,5 +591,7 @@ backend/files
*.db-wal
*.db-shm
.deploy-cache
!scripts/offline_python_linux_amd64/venv_linux_amd64/venv/
!scripts/offline_python_linux_amd64/venv_linux_amd64/venv/lib
scripts/deploy.sh
+201 -471
View File
@@ -1,5 +1,5 @@
#!/bin/bash
set -e
set -euo pipefail
# Complete deployment script: build and push to server
# Usage: ./scripts/deploy.sh [server_user@server_host] [deployment_path] [platform]
@@ -22,7 +22,7 @@ warning() { echo -e "${YELLOW}⚠${NC} $1"; }
error() { echo -e "${RED}✗${NC} $1"; }
step() { echo -e "${CYAN}${BOLD}→${NC} ${BOLD}$1${NC}"; }
substep() {
if [ "$2" = "-n" ]; then
if [ "${2:-}" = "-n" ]; then
echo -n -e " ${GREEN}•${NC} $1"
else
echo -e " ${GREEN}•${NC} $1"
@@ -64,6 +64,12 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
DEPLOYMENT_DIR="$PROJECT_ROOT/deployment"
ENV_FILE="$DEPLOYMENT_DIR/.env"
INPUT_HASH_TOOL="$SCRIPT_DIR/docker_inputs_hash.py"
LOCAL_CACHE_ROOT="$PROJECT_ROOT/.deploy-cache"
LOCAL_IMAGE_CACHE_DIR="$LOCAL_CACHE_ROOT/images"
# Ensure arrays exist even when build is skipped (nounset-safe).
declare -a BUILT_IMAGES
# Load .env file if it exists
if [ -f "$ENV_FILE" ]; then
@@ -102,6 +108,37 @@ if [ -n "$HOST_ARCH" ] && [ "$HOST_ARCH" = "$PLATFORM_ARCH" ]; then
USE_DOCKER_BUILD=true
fi
sanitize_ref() {
# Replace characters that are problematic for file paths.
# Example: "foo/bar:baz" -> "foo_bar__baz"
local s="$1"
s="${s//\//_}"
s="${s//:/__}"
s="${s//@/__at__}"
echo "$s"
}
read_file_if_exists() {
local p="$1"
if [ -f "$p" ]; then
cat "$p"
fi
}
# Docker's top-level image Id can differ for the same layers (OCI index vs image, BuildKit attestations).
# Compare a fingerprint of the final layer stack instead.
get_local_image_layer_fp() {
local img="$1"
docker image inspect -f '{{json .RootFS.Layers}}' "$img" 2>/dev/null \
| openssl dgst -sha256 2>/dev/null | awk '{print $2}' || true
}
compute_inputs_hash() {
local context="$1"
local dockerfile="$2"
python3 "$INPUT_HASH_TOOL" --context "$context" --dockerfile "$dockerfile"
}
# Check if server is provided
if [ -z "$SERVER" ]; then
error "Server not specified. Usage: $0 [user@host] [deployment_path] [platform]"
@@ -299,6 +336,12 @@ if ! COMPOSE_JSON=$(docker compose -f docker-compose.yml config --format json 2>
fi
BUILT_IMAGES=()
PUSHABLE_IMAGES=()
PUSHABLE_SERVICES=()
PUSHABLE_SERVICE_DOCKERFILES=()
PUSHABLE_SERVICE_CONTEXTS=()
PUSHABLE_SERVICE_TARGETS=()
PUSHABLE_IMAGE_INPUT_HASHES=()
for SERVICE in $SERVICES; do
if ! jq -e --arg s "$SERVICE" '(.services[$s].build // false) | type == "object"' <<< "$COMPOSE_JSON" >/dev/null 2>&1; then
@@ -307,8 +350,6 @@ for SERVICE in $SERVICES; do
IMAGE_TAG="${PROJECT_NAME}-${SERVICE}:latest"
substep "Building ${CYAN}$SERVICE${NC} -> ${CYAN}$IMAGE_TAG${NC}..."
DOCKERFILE_REL=$(jq -r --arg s "$SERVICE" '.services[$s].build.dockerfile // empty' <<< "$COMPOSE_JSON")
CONTEXT_REL=$(jq -r --arg s "$SERVICE" '.services[$s].build.context // empty' <<< "$COMPOSE_JSON")
# Multi-stage deployment/Dockerfile: without --target, the final stage (file_storage) is always tagged.
@@ -346,442 +387,100 @@ for SERVICE in $SERVICES; do
fi
fi
if [ "$USE_DOCKER_BUILD" = true ]; then
DOCKER_BUILD_ARGS=(build --platform "$PLATFORM" --file "$DOCKERFILE" --tag "$IMAGE_TAG")
if [ -n "$BUILD_TARGET" ]; then
DOCKER_BUILD_ARGS+=(--target "$BUILD_TARGET")
fi
DOCKER_BUILD_ARGS+=("$BUILD_CONTEXT")
if docker "${DOCKER_BUILD_ARGS[@]}"; then
echo -e " ${GREEN}✓${NC} Built ${CYAN}$SERVICE${NC}"
BUILT_IMAGES+=("$IMAGE_TAG")
echo ""
else
error "Build failed for $SERVICE"
exit 1
fi
else
# On macOS Docker Desktop, --load can hang for a long time at "sending tarball".
# Use the docker exporter explicitly to load into the local Docker daemon.
BUILDX_ARGS=(buildx build --platform "$PLATFORM" --file "$DOCKERFILE" --tag "$IMAGE_TAG" --output=type=docker)
if [ -n "$BUILD_TARGET" ]; then
BUILDX_ARGS+=(--target "$BUILD_TARGET")
fi
BUILDX_ARGS+=("$BUILD_CONTEXT")
if docker "${BUILDX_ARGS[@]}"; then
echo -e " ${GREEN}✓${NC} Built ${CYAN}$SERVICE${NC}"
BUILT_IMAGES+=("$IMAGE_TAG")
echo ""
else
error "Build failed for $SERVICE"
exit 1
fi
fi
true
if [ "$USE_DOCKER_BUILD" = false ]; then
true
fi
if [ "$USE_DOCKER_BUILD" = true ]; then
true
fi
if [ "$USE_DOCKER_BUILD" = false ]; then
true
fi
if [ "$USE_DOCKER_BUILD" = true ]; then
true
fi
if [ "$USE_DOCKER_BUILD" = false ]; then
true
fi
if [ "$USE_DOCKER_BUILD" = true ]; then
true
fi
if [ "$USE_DOCKER_BUILD" = false ]; then
true
fi
if [ "$USE_DOCKER_BUILD" = true ]; then
true
fi
if [ "$USE_DOCKER_BUILD" = false ]; then
true
fi
if [ "$USE_DOCKER_BUILD" = true ]; then
true
fi
if [ "$USE_DOCKER_BUILD" = false ]; then
true
fi
if [ "$USE_DOCKER_BUILD" = true ]; then
true
fi
if [ "$USE_DOCKER_BUILD" = false ]; then
true
fi
if [ "$USE_DOCKER_BUILD" = true ]; then
true
fi
if [ "$USE_DOCKER_BUILD" = false ]; then
true
fi
if [ "$USE_DOCKER_BUILD" = true ]; then
true
fi
if [ "$USE_DOCKER_BUILD" = false ]; then
true
fi
if [ "$USE_DOCKER_BUILD" = true ]; then
true
fi
if [ "$USE_DOCKER_BUILD" = false ]; then
true
fi
if [ "$USE_DOCKER_BUILD" = true ]; then
true
fi
if [ "$USE_DOCKER_BUILD" = false ]; then
true
fi
if [ "$USE_DOCKER_BUILD" = true ]; then
true
fi
if [ "$USE_DOCKER_BUILD" = false ]; then
true
fi
if [ "$USE_DOCKER_BUILD" = true ]; then
true
fi
if [ "$USE_DOCKER_BUILD" = false ]; then
true
fi
if [ "$USE_DOCKER_BUILD" = true ]; then
true
fi
if [ "$USE_DOCKER_BUILD" = false ]; then
true
fi
if [ "$USE_DOCKER_BUILD" = true ]; then
true
fi
if [ "$USE_DOCKER_BUILD" = false ]; then
true
fi
if [ "$USE_DOCKER_BUILD" = true ]; then
true
fi
if [ "$USE_DOCKER_BUILD" = false ]; then
true
fi
if [ "$USE_DOCKER_BUILD" = true ]; then
true
fi
if [ "$USE_DOCKER_BUILD" = false ]; then
true
fi
if [ "$USE_DOCKER_BUILD" = true ]; then
true
fi
if [ "$USE_DOCKER_BUILD" = false ]; then
true
fi
if [ "$USE_DOCKER_BUILD" = true ]; then
true
fi
if [ "$USE_DOCKER_BUILD" = false ]; then
true
fi
if [ "$USE_DOCKER_BUILD" = true ]; then
true
fi
if [ "$USE_DOCKER_BUILD" = false ]; then
true
fi
if [ "$USE_DOCKER_BUILD" = true ]; then
true
fi
if [ "$USE_DOCKER_BUILD" = false ]; then
true
fi
if [ "$USE_DOCKER_BUILD" = true ]; then
true
fi
if [ "$USE_DOCKER_BUILD" = false ]; then
true
fi
if [ "$USE_DOCKER_BUILD" = true ]; then
true
fi
if [ "$USE_DOCKER_BUILD" = false ]; then
true
fi
if [ "$USE_DOCKER_BUILD" = true ]; then
true
fi
if [ "$USE_DOCKER_BUILD" = false ]; then
true
fi
if [ "$USE_DOCKER_BUILD" = true ]; then
true
fi
if [ "$USE_DOCKER_BUILD" = false ]; then
true
fi
if [ "$USE_DOCKER_BUILD" = true ]; then
true
fi
if [ "$USE_DOCKER_BUILD" = false ]; then
true
fi
if [ "$USE_DOCKER_BUILD" = true ]; then
true
fi
if [ "$USE_DOCKER_BUILD" = false ]; then
true
fi
if [ "$USE_DOCKER_BUILD" = true ]; then
true
fi
if [ "$USE_DOCKER_BUILD" = false ]; then
true
fi
if [ "$USE_DOCKER_BUILD" = true ]; then
true
fi
if [ "$USE_DOCKER_BUILD" = false ]; then
true
fi
if [ "$USE_DOCKER_BUILD" = true ]; then
true
fi
if [ "$USE_DOCKER_BUILD" = false ]; then
true
fi
if [ "$USE_DOCKER_BUILD" = true ]; then
true
fi
if [ "$USE_DOCKER_BUILD" = false ]; then
true
fi
if [ "$USE_DOCKER_BUILD" = true ]; then
true
fi
if [ "$USE_DOCKER_BUILD" = false ]; then
true
fi
if [ "$USE_DOCKER_BUILD" = true ]; then
true
fi
if [ "$USE_DOCKER_BUILD" = false ]; then
true
fi
if [ "$USE_DOCKER_BUILD" = true ]; then
true
fi
if [ "$USE_DOCKER_BUILD" = false ]; then
true
fi
if [ "$USE_DOCKER_BUILD" = true ]; then
true
fi
if [ "$USE_DOCKER_BUILD" = false ]; then
true
fi
if [ "$USE_DOCKER_BUILD" = true ]; then
true
fi
if [ "$USE_DOCKER_BUILD" = false ]; then
true
fi
if [ "$USE_DOCKER_BUILD" = true ]; then
true
fi
if [ "$USE_DOCKER_BUILD" = false ]; then
true
fi
if [ "$USE_DOCKER_BUILD" = true ]; then
true
fi
if [ "$USE_DOCKER_BUILD" = false ]; then
true
fi
if [ "$USE_DOCKER_BUILD" = true ]; then
true
fi
if [ "$USE_DOCKER_BUILD" = false ]; then
true
fi
if [ "$USE_DOCKER_BUILD" = true ]; then
true
fi
if [ "$USE_DOCKER_BUILD" = false ]; then
true
fi
if [ "$USE_DOCKER_BUILD" = true ]; then
true
fi
if [ "$USE_DOCKER_BUILD" = false ]; then
true
fi
if [ "$USE_DOCKER_BUILD" = true ]; then
true
fi
if [ "$USE_DOCKER_BUILD" = false ]; then
true
fi
if [ "$USE_DOCKER_BUILD" = true ]; then
true
fi
if [ "$USE_DOCKER_BUILD" = false ]; then
true
fi
if [ "$USE_DOCKER_BUILD" = true ]; then
true
if [ ! -f "$INPUT_HASH_TOOL" ]; then
error "Missing $INPUT_HASH_TOOL (needed for dependency hashing)"
exit 1
fi
if [ "$USE_DOCKER_BUILD" = false ]; then
true
fi
if [ "$USE_DOCKER_BUILD" = true ]; then
true
fi
if [ "$USE_DOCKER_BUILD" = false ]; then
true
fi
if [ "$USE_DOCKER_BUILD" = true ]; then
true
fi
if [ "$USE_DOCKER_BUILD" = false ]; then
true
fi
if [ "$USE_DOCKER_BUILD" = true ]; then
true
fi
if [ "$USE_DOCKER_BUILD" = false ]; then
true
fi
if [ "$USE_DOCKER_BUILD" = true ]; then
true
fi
if [ "$USE_DOCKER_BUILD" = false ]; then
true
fi
if [ "$USE_DOCKER_BUILD" = true ]; then
true
fi
if [ "$USE_DOCKER_BUILD" = false ]; then
true
fi
if [ "$USE_DOCKER_BUILD" = true ]; then
true
fi
if [ "$USE_DOCKER_BUILD" = false ]; then
true
fi
INPUT_HASH=$(compute_inputs_hash "$BUILD_CONTEXT" "$DOCKERFILE")
true
PUSHABLE_SERVICES+=("$SERVICE")
PUSHABLE_IMAGES+=("$IMAGE_TAG")
PUSHABLE_SERVICE_DOCKERFILES+=("$DOCKERFILE")
PUSHABLE_SERVICE_CONTEXTS+=("$BUILD_CONTEXT")
PUSHABLE_SERVICE_TARGETS+=("$BUILD_TARGET")
PUSHABLE_IMAGE_INPUT_HASHES+=("$INPUT_HASH")
done
success "Build complete! ${#BUILT_IMAGES[@]} image(s) ready"
mkdir -p "$LOCAL_IMAGE_CACHE_DIR"
SERVICES_TO_BUILD=()
IMAGES_TO_BUILD=()
INPUT_HASHES_TO_BUILD=()
DOCKERFILES_TO_BUILD=()
CONTEXTS_TO_BUILD=()
TARGETS_TO_BUILD=()
for i in "${!PUSHABLE_SERVICES[@]}"; do
SERVICE="${PUSHABLE_SERVICES[$i]}"
IMAGE_TAG="${PUSHABLE_IMAGES[$i]}"
INPUT_HASH="${PUSHABLE_IMAGE_INPUT_HASHES[$i]}"
DOCKERFILE="${PUSHABLE_SERVICE_DOCKERFILES[$i]}"
BUILD_CONTEXT="${PUSHABLE_SERVICE_CONTEXTS[$i]}"
BUILD_TARGET="${PUSHABLE_SERVICE_TARGETS[$i]}"
CACHE_KEY="$(sanitize_ref "$IMAGE_TAG")"
CACHE_FILE="$LOCAL_IMAGE_CACHE_DIR/$CACHE_KEY/input.sha256"
PREV_HASH="$(read_file_if_exists "$CACHE_FILE")"
LOCAL_FP="$(get_local_image_layer_fp "$IMAGE_TAG")"
if [ -n "$PREV_HASH" ] && [ "$PREV_HASH" = "$INPUT_HASH" ] && [ -n "$LOCAL_FP" ]; then
continue
fi
SERVICES_TO_BUILD+=("$SERVICE")
IMAGES_TO_BUILD+=("$IMAGE_TAG")
INPUT_HASHES_TO_BUILD+=("$INPUT_HASH")
DOCKERFILES_TO_BUILD+=("$DOCKERFILE")
CONTEXTS_TO_BUILD+=("$BUILD_CONTEXT")
TARGETS_TO_BUILD+=("$BUILD_TARGET")
done
if [ ${#SERVICES_TO_BUILD[@]} -eq 0 ]; then
success "Build skipped (no Docker inputs changed)"
else
step "Building ${#SERVICES_TO_BUILD[@]} service(s)"
if [ "$USE_DOCKER_BUILD" = true ]; then
# Same-arch fast path: build with Docker Compose (requested).
# Ensure the project name is stable so the resulting tags match our expectations.
COMPOSE_PROJECT_NAME="$PROJECT_NAME" docker compose -f docker-compose.yml --profile production build "${SERVICES_TO_BUILD[@]}"
else
for i in "${!SERVICES_TO_BUILD[@]}"; do
SERVICE="${SERVICES_TO_BUILD[$i]}"
IMAGE_TAG="${IMAGES_TO_BUILD[$i]}"
DOCKERFILE="${DOCKERFILES_TO_BUILD[$i]}"
BUILD_CONTEXT="${CONTEXTS_TO_BUILD[$i]}"
BUILD_TARGET="${TARGETS_TO_BUILD[$i]}"
substep "Building ${CYAN}$SERVICE${NC} -> ${CYAN}$IMAGE_TAG${NC}..."
# On macOS Docker Desktop, --load can hang for a long time at "sending tarball".
# Use the docker exporter explicitly to load into the local Docker daemon.
BUILDX_ARGS=(buildx build --platform "$PLATFORM" --file "$DOCKERFILE" --tag "$IMAGE_TAG" --output=type=docker --provenance=false --sbom=false)
if [ -n "$BUILD_TARGET" ]; then
BUILDX_ARGS+=(--target "$BUILD_TARGET")
fi
BUILDX_ARGS+=("$BUILD_CONTEXT")
if ! docker "${BUILDX_ARGS[@]}"; then
error "Build failed for $SERVICE"
exit 1
fi
done
fi
for i in "${!SERVICES_TO_BUILD[@]}"; do
IMAGE_TAG="${IMAGES_TO_BUILD[$i]}"
INPUT_HASH="${INPUT_HASHES_TO_BUILD[$i]}"
CACHE_KEY="$(sanitize_ref "$IMAGE_TAG")"
mkdir -p "$LOCAL_IMAGE_CACHE_DIR/$CACHE_KEY"
printf "%s" "$INPUT_HASH" > "$LOCAL_IMAGE_CACHE_DIR/$CACHE_KEY/input.sha256"
BUILT_IMAGES+=("$IMAGE_TAG")
done
success "Build complete! ${#BUILT_IMAGES[@]} image(s) built"
fi
# ============================================================================
# DEPLOY PHASE
@@ -837,31 +536,36 @@ fi
BUILT_COUNT=${#BUILT_IMAGES[@]}
MATCHING_BUILT=0
MISSING_FROM_DETECTED=()
for BI in "${BUILT_IMAGES[@]}"; do
found=false
for DI in "${PUSH_IMAGES[@]}"; do
if [ "$BI" = "$DI" ]; then
found=true
break
# Bash 3.2 + set -u: "${arr[@]}" on an empty array can error; guard with length.
if ((${#BUILT_IMAGES[@]} > 0)); then
for BI in "${BUILT_IMAGES[@]}"; do
found=false
for DI in "${PUSH_IMAGES[@]}"; do
if [ "$BI" = "$DI" ]; then
found=true
break
fi
done
if [ "$found" = true ]; then
MATCHING_BUILT=$((MATCHING_BUILT + 1))
else
MISSING_FROM_DETECTED+=("$BI")
fi
done
if [ "$found" = true ]; then
MATCHING_BUILT=$((MATCHING_BUILT + 1))
else
MISSING_FROM_DETECTED+=("$BI")
fi
done
fi
# Also list push images that weren't built locally (these are likely prebuilt local images)
NOT_BUILT_DETECTED=()
for DI in "${PUSH_IMAGES[@]}"; do
built=false
for BI in "${BUILT_IMAGES[@]}"; do
if [ "$DI" = "$BI" ]; then
built=true
break
fi
done
if ((${#BUILT_IMAGES[@]} > 0)); then
for BI in "${BUILT_IMAGES[@]}"; do
if [ "$DI" = "$BI" ]; then
built=true
break
fi
done
fi
if [ "$built" = false ]; then
NOT_BUILT_DETECTED+=("$DI")
fi
@@ -883,33 +587,59 @@ if [ ${#PUSH_IMAGES[@]} -eq 0 ] && [ ${#EXTERNAL_IMAGES[@]} -eq 0 ]; then
error "No images found in docker-compose.yml or built locally for project ${PROJECT_NAME}"
fi
# Pre-pull unregistry image if needed
UNREGISTRY_IMAGE="ghcr.io/psviderski/unregistry"
if ! ssh "$SERVER" "sudo docker images --format '{{.Repository}}:{{.Tag}}' | grep -q '^${UNREGISTRY_IMAGE}$'" 2>/dev/null; then
substep "Pulling unregistry image (one-time setup)..."
ssh -tt "$SERVER" "sudo docker pull ${UNREGISTRY_IMAGE}"
# Transfer images — only what was rebuilt this run (BUILT_IMAGES). No remote hash/fingerprint checks.
step "Transferring images"
IMAGES_TO_PUSH=()
if ((${#BUILT_IMAGES[@]} > 0)); then
for PI in "${PUSH_IMAGES[@]}"; do
for BI in "${BUILT_IMAGES[@]}"; do
if [ "$PI" = "$BI" ]; then
IMAGES_TO_PUSH+=("$PI")
break
fi
done
done
fi
# Transfer images
step "Transferring images"
for IMAGE in "${PUSH_IMAGES[@]}"; do
substep "Pushing ${CYAN}$IMAGE${NC}..."
if docker pussh "$IMAGE" "$SERVER"; then
echo ""
else
echo -e " ${RED}✗${NC} Failed to push ${CYAN}$IMAGE${NC}"
echo ""
exit 1
if [ ${#IMAGES_TO_PUSH[@]} -eq 0 ]; then
success "Skipping image push (nothing was rebuilt this run)"
else
UNREGISTRY_IMAGE="ghcr.io/psviderski/unregistry"
if ! ssh "$SERVER" "sudo docker images --format '{{.Repository}}:{{.Tag}}' | grep -q '^${UNREGISTRY_IMAGE}$'" 2>/dev/null; then
substep "Pulling unregistry image (one-time setup)..."
if [ -n "${SUDO_PASSWORD:-}" ]; then
ssh "$SERVER" "echo '$SUDO_PASSWORD' | sudo -S -p '' docker pull ${UNREGISTRY_IMAGE}"
else
ssh "$SERVER" "sudo docker pull ${UNREGISTRY_IMAGE}"
fi
fi
done
for IMAGE in "${IMAGES_TO_PUSH[@]}"; do
substep "Pushing ${CYAN}$IMAGE${NC}..."
if ! docker pussh "$IMAGE" "$SERVER"; then
echo -e " ${RED}✗${NC} Failed to push ${CYAN}$IMAGE${NC}"
echo ""
exit 1
fi
echo ""
done
fi
# Pull external images directly on the server (no pussh)
if [ ${#EXTERNAL_IMAGES[@]} -gt 0 ]; then
step "Pulling external images on server"
for IMAGE in "${EXTERNAL_IMAGES[@]}"; do
substep "Pulling ${CYAN}$IMAGE${NC}..."
# Allocate a TTY and do not redirect output so failures are visible.
if ssh -tt "$SERVER" "sudo docker pull $(printf '%q' "$IMAGE")"; then
QIMG=$(printf '%q' "$IMAGE")
if [ -n "${SUDO_PASSWORD:-}" ]; then
if ssh "$SERVER" "echo '$SUDO_PASSWORD' | sudo -S -p '' docker pull $QIMG"; then
echo ""
else
echo -e " ${RED}✗${NC} Failed to pull ${CYAN}$IMAGE${NC} on server"
echo ""
exit 1
fi
elif ssh "$SERVER" "sudo docker pull $QIMG"; then
echo ""
else
echo -e " ${RED}✗${NC} Failed to pull ${CYAN}$IMAGE${NC} on server"
scripts/docker_inputs_hash.py
+369
View File
@@ -0,0 +1,369 @@
#!/usr/bin/env python3
from __future__ import annotations
import argparse
import fnmatch
import hashlib
import json
import os
import shlex
import sys
from dataclasses import dataclass
from pathlib import Path
def _sha256_bytes(data: bytes) -> str:
h = hashlib.sha256()
h.update(data)
return h.hexdigest()
def _sha256_file(path: Path) -> str:
h = hashlib.sha256()
with path.open("rb") as f:
for chunk in iter(lambda: f.read(1024 * 1024), b""):
h.update(chunk)
return h.hexdigest()
def _read_text(path: Path) -> str:
return path.read_text(encoding="utf-8", errors="replace")
@dataclass(frozen=True)
class DockerIgnoreRule:
pattern: str
negated: bool
anchored: bool
directory_only: bool
def _read_dockerignore_rules(context: Path) -> list[DockerIgnoreRule]:
p = context / ".dockerignore"
if not p.exists() or not p.is_file():
return []
rules: list[DockerIgnoreRule] = []
for raw in _read_text(p).splitlines():
line = raw.strip()
if not line or line.startswith("#"):
continue
negated = line.startswith("!")
if negated:
line = line[1:].lstrip()
anchored = line.startswith("/")
if anchored:
line = line[1:]
directory_only = line.endswith("/")
if directory_only:
line = line[:-1]
if not line:
continue
rules.append(
DockerIgnoreRule(
pattern=line,
negated=negated,
anchored=anchored,
directory_only=directory_only,
)
)
return rules
def _dockerignore_matches(rule: DockerIgnoreRule, rel_posix: str, is_dir: bool) -> bool:
if rule.directory_only and not is_dir:
return False
rel = rel_posix.lstrip("./")
if rule.anchored:
# Anchored to context root.
candidates = [rel]
else:
# Unanchored patterns match anywhere: try both full rel and basename.
base = rel.rsplit("/", 1)[-1]
candidates = [rel, base]
# Dockerignore supports ** globs; fnmatch handles this well enough for our use.
for c in candidates:
if fnmatch.fnmatch(c, rule.pattern):
return True
# Also allow matching directory prefixes for patterns like "dist" against "foo/dist/bar".
if not rule.anchored and "/" in rel:
if fnmatch.fnmatch(rel, f"*/{rule.pattern}") or fnmatch.fnmatch(rel, f"**/{rule.pattern}"):
return True
return False
def _is_ignored_by_dockerignore(rules: list[DockerIgnoreRule], rel_posix: str, is_dir: bool) -> bool:
ignored = False
for r in rules:
if _dockerignore_matches(r, rel_posix=rel_posix, is_dir=is_dir):
ignored = not r.negated
return ignored
def _dockerfile_logical_lines(dockerfile_text: str) -> list[str]:
"""
Join backslash-continued lines and drop full-line comments.
"""
out: list[str] = []
buf: list[str] = []
for raw in dockerfile_text.splitlines():
line = raw.rstrip()
if not buf:
stripped = line.lstrip()
if stripped.startswith("#") or stripped == "":
continue
buf.append(line)
if line.endswith("\\"):
buf[-1] = buf[-1][:-1].rstrip()
continue
joined = " ".join(x.strip() for x in buf if x.strip())
buf = []
if joined:
out.append(joined)
if buf:
joined = " ".join(x.strip() for x in buf if x.strip())
if joined:
out.append(joined)
return out
@dataclass(frozen=True)
class CopyAdd:
sources: tuple[str, ...]
from_stage: bool
def _parse_copy_add_args_shellform(args: list[str]) -> CopyAdd | None:
# flags: --from=, --chown=, --chmod=, --link, --parents, --exclude=... etc.
from_stage = False
rest: list[str] = []
for a in args:
if a.startswith("--from=") or a == "--from":
from_stage = True
continue
if a.startswith("--"):
continue
rest.append(a)
if len(rest) < 2:
return None
# last is dest
srcs = tuple(rest[:-1])
return CopyAdd(sources=srcs, from_stage=from_stage)
def _parse_copy_add_args_jsonform(json_text: str) -> CopyAdd | None:
try:
arr = json.loads(json_text)
except Exception:
return None
if not isinstance(arr, list) or len(arr) < 2:
return None
# last is dest
srcs = tuple(x for x in arr[:-1] if isinstance(x, str))
if not srcs:
return None
return CopyAdd(sources=srcs, from_stage=False)
def _parse_copy_add(line: str) -> CopyAdd | None:
upper = line.lstrip().upper()
if not (upper.startswith("COPY ") or upper.startswith("ADD ")):
return None
# Keep original casing for paths.
keyword, rest = line.split(None, 1)
rest = rest.strip()
# JSON form starts with '['
if rest.startswith("["):
parsed = _parse_copy_add_args_jsonform(rest)
if parsed:
return parsed
return None
# shell form
try:
parts = shlex.split(rest, posix=True)
except Exception:
return None
return _parse_copy_add_args_shellform(parts)
def _looks_like_remote(src: str) -> bool:
s = src.lower()
return s.startswith("http://") or s.startswith("https://")
def _is_glob(p: str) -> bool:
return any(ch in p for ch in ["*", "?", "["])
def _iter_files_under(path: Path) -> list[Path]:
if not path.exists():
return []
if path.is_file():
return [path]
files: list[Path] = []
for root, _, filenames in os.walk(path):
for name in filenames:
files.append(Path(root) / name)
return files
def _collect_sources(context: Path, dockerfile_path: Path) -> list[Path]:
text = _read_text(dockerfile_path)
logical = _dockerfile_logical_lines(text)
dockerignore_rules = _read_dockerignore_rules(context)
paths: list[Path] = []
for ln in logical:
parsed = _parse_copy_add(ln)
if not parsed:
continue
if parsed.from_stage:
continue
for src in parsed.sources:
if _looks_like_remote(src):
continue
if src.startswith("/"):
# Absolute COPY sources aren't valid for local context; ignore to avoid surprises.
continue
# Docker allows ".", "./foo", etc.
src_norm = src.lstrip("./")
if src_norm == "":
src_norm = "."
if _is_glob(src_norm):
# Expand within context
for root, _, filenames in os.walk(context):
root_p = Path(root)
rel_root = root_p.relative_to(context).as_posix()
for fn in filenames:
rel = f"{rel_root}/{fn}" if rel_root != "." else fn
if _is_ignored_by_dockerignore(dockerignore_rules, rel_posix=rel, is_dir=False):
continue
if fnmatch.fnmatch(rel, src_norm) or fnmatch.fnmatch(fn, src_norm):
paths.append(context / rel)
continue
p = (context / src_norm).resolve()
# Ensure stays within context
try:
p.relative_to(context.resolve())
except Exception:
continue
for fp in _iter_files_under(p):
try:
rel = fp.resolve().relative_to(context.resolve()).as_posix()
except Exception:
continue
if _is_ignored_by_dockerignore(dockerignore_rules, rel_posix=rel, is_dir=fp.is_dir()):
continue
paths.append(fp)
# Always include the Dockerfile itself (and preserve stable ordering via sort later)
paths.append(dockerfile_path.resolve())
return paths
def compute_inputs_hash(context: Path, dockerfile_path: Path) -> str:
files = _collect_sources(context=context, dockerfile_path=dockerfile_path)
# Deduplicate by resolved path
uniq: dict[str, Path] = {}
for p in files:
uniq[str(p)] = p
# Stable sort by path relative to context when possible, else absolute
ctx_resolved = context.resolve()
def sort_key(p: Path) -> str:
try:
return p.resolve().relative_to(ctx_resolved).as_posix()
except Exception:
return p.resolve().as_posix()
sorted_files = sorted(uniq.values(), key=sort_key)
h = hashlib.sha256()
for p in sorted_files:
rp: str
try:
rp = p.resolve().relative_to(ctx_resolved).as_posix()
except Exception:
rp = p.resolve().as_posix()
h.update(rp.encode("utf-8", errors="strict"))
h.update(b"\0")
if p.is_file():
h.update(_sha256_file(p).encode("ascii"))
else:
h.update(b"NONFILE")
h.update(b"\n")
return h.hexdigest()
def compute_inputs_debug(context: Path, dockerfile_path: Path) -> tuple[str, list[tuple[str, str]]]:
"""
Returns (inputs_hash, [(rel_path, sha256_of_file_contents), ...]) with dockerignore applied.
"""
files = _collect_sources(context=context, dockerfile_path=dockerfile_path)
uniq: dict[str, Path] = {}
for p in files:
uniq[str(p.resolve())] = p.resolve()
ctx_resolved = context.resolve()
def rel_or_abs(p: Path) -> str:
try:
return p.resolve().relative_to(ctx_resolved).as_posix()
except Exception:
return p.resolve().as_posix()
sorted_files = sorted(uniq.values(), key=lambda p: rel_or_abs(p))
items: list[tuple[str, str]] = []
for p in sorted_files:
rp = rel_or_abs(p)
if p.is_file():
items.append((rp, _sha256_file(p)))
else:
items.append((rp, "NONFILE"))
return compute_inputs_hash(context=context, dockerfile_path=dockerfile_path), items
def main() -> int:
ap = argparse.ArgumentParser()
ap.add_argument("--context", required=True, help="Build context directory")
ap.add_argument("--dockerfile", required=True, help="Dockerfile path")
ap.add_argument(
"--debug-list",
action="store_true",
help="Print the included file list (relpath|sha256) to stderr",
)
args = ap.parse_args()
context = Path(args.context).resolve()
dockerfile = Path(args.dockerfile).resolve()
if not context.exists() or not context.is_dir():
print(f"Context not found or not a directory: {context}", file=sys.stderr)
return 2
if not dockerfile.exists() or not dockerfile.is_file():
print(f"Dockerfile not found: {dockerfile}", file=sys.stderr)
return 2
if args.debug_list:
h, items = compute_inputs_debug(context=context, dockerfile_path=dockerfile)
for rp, sh in items:
print(f"{rp}|{sh}", file=sys.stderr)
print(h)
else:
print(compute_inputs_hash(context=context, dockerfile_path=dockerfile))
return 0
if __name__ == "__main__":
raise SystemExit(main())