FromChat/web
1
0
Fork 0
mirror of https://github.com/fromchat-messenger/web.git synced 2026-09-22 19:15:08 +03:00
Code Issues Packages Projects Releases Wiki Activity

Rewrite deployment script in Python

Browse Source
This commit is contained in:
denis0001-dev
2026-04-02 11:02:06 +03:00
Unverified
parent 32e186213e
commit be7da286d4
12 changed files with 1090 additions and 809 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/requirements.txt
+1
View File
@@ -1,3 +1,4 @@
python-dotenv>=1.0.1
PyJWT>=2.8.0 PyJWT>=2.8.0
fastapi[standard]>=0.116.1 fastapi[standard]>=0.116.1
pydantic>=2.11.7 pydantic>=2.11.7
scripts/deploy.sh
+3 -809
View File
@@ -1,810 +1,4 @@
#!/bin/bash #!/usr/bin/env bash
set -euo pipefail set -euo pipefail
cd "$(dirname "$0")"
# Complete deployment script: build and push to server exec ../.venv/bin/python3 deploy/main.py "$@"
# Usage: ./scripts/deploy.sh [server_user@server_host] [deployment_path] [platform]
# Example: ./scripts/deploy.sh user@example.com /home/user/fromchat linux/arm64
# Colors
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
CYAN='\033[0;36m'
MAGENTA='\033[0;35m'
NC='\033[0m' # No Color
BOLD='\033[1m'
# Helper functions
info() { echo -e "${BLUE}ℹ${NC} $1"; }
success() { echo -e "${GREEN}✓${NC} $1"; }
warning() { echo -e "${YELLOW}⚠${NC} $1"; }
error() { echo -e "${RED}✗${NC} $1"; }
step() { echo -e "${CYAN}${BOLD}→${NC} ${BOLD}$1${NC}"; }
substep() {
if [ "${2:-}" = "-n" ]; then
echo -n -e " ${GREEN}•${NC} $1"
else
echo -e " ${GREEN}•${NC} $1"
fi
}
echo -e "${MAGENTA}${BOLD}🚀 Deployment${NC}\n"
read_password() {
local password=""
local char
local old_stty
old_stty=$(stty -g 2>/dev/null)
stty -echo 2>/dev/null
while IFS= read -rs -n 1 char; do
if [ -z "$char" ]; then
break
fi
if [ "$char" = $'\177' ] || [ "$char" = $'\b' ]; then
if [ ${#password} -gt 0 ]; then
password="${password%?}"
printf "\b \b" >&2
fi
else
password+="$char"
printf "*" >&2
fi
done
stty "$old_stty" 2>/dev/null
echo "" >&2
echo "$password"
}
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
DEPLOYMENT_DIR="$PROJECT_ROOT/deployment"
ENV_FILE="$DEPLOYMENT_DIR/.env"
INPUT_HASH_TOOL="$SCRIPT_DIR/docker_inputs_hash.py"
LOCAL_CACHE_ROOT="$PROJECT_ROOT/.deploy-cache"
LOCAL_IMAGE_CACHE_DIR="$LOCAL_CACHE_ROOT/images"
# Ensure arrays exist even when build is skipped (nounset-safe).
declare -a BUILT_IMAGES
# Load .env file if it exists
if [ -f "$ENV_FILE" ]; then
# Export variables from .env file (ignore comments and empty lines)
set -a
while IFS= read -r line || [ -n "$line" ]; do
# Skip comments and empty lines
case "$line" in
\#*|'') continue ;;
*)
# Export the variable
export "$line" 2>/dev/null || true
;;
esac
done < "$ENV_FILE"
set +a
fi
# Read server from environment variable (from .env), command line argument, or fallback
SERVER="${1:-${DEPLOYMENT_SERVER:-}}"
REPO_NAME="FromChat"
DEPLOY_PATH="~/actions-runner/_work/$REPO_NAME/$REPO_NAME"
PLATFORM="linux/arm64"
# Prefer plain `docker build` when target arch matches host arch.
# This avoids Docker Desktop buildx export/load issues and is faster for same-arch builds.
HOST_ARCH_RAW="$(uname -m 2>/dev/null || echo "")"
case "$HOST_ARCH_RAW" in
arm64|aarch64) HOST_ARCH="arm64" ;;
x86_64|amd64) HOST_ARCH="amd64" ;;
*) HOST_ARCH="$HOST_ARCH_RAW" ;;
esac
PLATFORM_ARCH="${PLATFORM##*/}"
USE_DOCKER_BUILD=false
if [ -n "$HOST_ARCH" ] && [ "$HOST_ARCH" = "$PLATFORM_ARCH" ]; then
USE_DOCKER_BUILD=true
fi
sanitize_ref() {
# Replace characters that are problematic for file paths.
# Example: "foo/bar:baz" -> "foo_bar__baz"
local s="$1"
s="${s//\//_}"
s="${s//:/__}"
s="${s//@/__at__}"
echo "$s"
}
read_file_if_exists() {
local p="$1"
if [ -f "$p" ]; then
cat "$p"
fi
}
# Docker's top-level image Id can differ for the same layers (OCI index vs image, BuildKit attestations).
# Compare a fingerprint of the final layer stack instead.
get_local_image_layer_fp() {
local img="$1"
docker image inspect -f '{{json .RootFS.Layers}}' "$img" 2>/dev/null \
| openssl dgst -sha256 2>/dev/null | awk '{print $2}' || true
}
compute_inputs_hash() {
local context="$1"
local dockerfile="$2"
python3 "$INPUT_HASH_TOOL" --context "$context" --dockerfile "$dockerfile"
}
# Check if server is provided
if [ -z "$SERVER" ]; then
error "Server not specified. Usage: $0 [user@host] [deployment_path] [platform]"
echo " Or set DEPLOYMENT_SERVER in $ENV_FILE or as an environment variable"
echo ""
echo "Example:"
echo " $0 user@example.com /home/user/fromchat linux/arm64"
echo " Or add to $ENV_FILE: DEPLOYMENT_SERVER=user@example.com"
echo " Or: DEPLOYMENT_SERVER=user@example.com $0"
exit 1
fi
# ============================================================================
# SSH AUTHENTICATION
# ============================================================================
step "Authentication"
SSH_KEY_FILE="$HOME/.ssh/id_rsa"
SSH_KEY_PUB_FILE="$SSH_KEY_FILE.pub"
# Ensure ssh-agent is running
if [ -z "$SSH_AUTH_SOCK" ]; then
eval "$(ssh-agent -s)" > /dev/null 2>&1
fi
# Check if SSH key exists
if [ ! -f "$SSH_KEY_FILE" ]; then
error "SSH key not found at $SSH_KEY_FILE"
echo " Please generate an SSH key pair first:"
echo " ssh-keygen -t rsa -b 4096 -C 'your_email@example.com'"
exit 1
fi
# Add SSH key to agent if not already loaded
KEY_LOADED=false
if ssh-add -l > /dev/null 2>&1; then
# Check if this specific key is loaded by trying to match the public key
KEY_FINGERPRINT=$(ssh-keygen -lf "$SSH_KEY_FILE" 2>/dev/null | awk '{print $2}')
if [ -n "$KEY_FINGERPRINT" ] && ssh-add -l 2>/dev/null | grep -q "$KEY_FINGERPRINT"; then
KEY_LOADED=true
fi
fi
if [ "$KEY_LOADED" = false ]; then
substep "Adding SSH key to agent..."
if ! ssh-add "$SSH_KEY_FILE" 2>/dev/null; then
error "Failed to add SSH key to agent. Check your key passphrase."
exit 1
fi
fi
# Require key-based SSH auth; do not attempt to copy keys automatically.
if ! ssh -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=no "$SERVER" "echo 'SSH key works'" >/dev/null 2>&1; then
error "SSH key authentication failed for $SERVER"
echo " Copy your public key to the server, then re-run deploy:"
echo " ssh-copy-id -i \"$SSH_KEY_PUB_FILE\" \"$SERVER\""
echo ""
echo " Or manually append this key to ~/.ssh/authorized_keys on the server:"
echo " $(cat "$SSH_KEY_PUB_FILE")"
exit 1
fi
# ============================================================================
# SUDO AUTHENTICATION
# ============================================================================
SUDO_PASSWORD=""
# Prompt for sudo password (optional; leave blank for passwordless sudo)
if [ -z "$SUDO_PASSWORD" ]; then
while true; do
substep "Sudo password: " -n
SUDO_PASSWORD=$(read_password)
if [ -z "$SUDO_PASSWORD" ]; then
warning "No password provided - assuming passwordless sudo"
break
fi
if echo "$SUDO_PASSWORD" | ssh "$SERVER" "sudo -S -v" > /dev/null 2>&1; then
export SUDO_PASSWORD
break
else
echo -n " " && error "Invalid password, please try again"
fi
done
fi
# ============================================================================
# BUILD PHASE
# ============================================================================
echo -e "\n${MAGENTA}${BOLD}🔨 Building Docker images${NC}\n"
# Determine project name
if [ -n "$SERVER" ]; then
COMPOSE_DIR=$(ssh "$SERVER" "dirname $DEPLOY_PATH/deployment/docker-compose.yml" 2>/dev/null || echo "$DEPLOY_PATH/deployment")
PROJECT_NAME=$(ssh "$SERVER" "basename $COMPOSE_DIR" 2>/dev/null || echo "deployment")
else
PROJECT_NAME=$(basename "$DEPLOYMENT_DIR")
fi
# Check if Docker daemon is running
check_docker_daemon() {
docker info > /dev/null 2>&1
}
# Start Docker Desktop
start_docker_desktop() {
substep "Starting Docker Desktop..."
if ! docker desktop start > /dev/null 2>&1; then
return 1
fi
# Wait for Docker to be ready (max 60 seconds)
substep "Waiting for Docker to start..." -n
local max_wait=60
local waited=0
while [ $waited -lt $max_wait ]; do
if check_docker_daemon; then
echo ""
return 0
fi
sleep 2
waited=$((waited + 2))
echo -n "."
done
echo ""
return 1
}
# Check Docker daemon
if ! check_docker_daemon; then
warning "Docker daemon is not running"
if ! start_docker_desktop; then
error "Failed to start Docker Desktop. Please start it manually and try again."
fi
fi
if [ "$USE_DOCKER_BUILD" = false ]; then
# Check buildx
if ! docker buildx version > /dev/null 2>&1; then
error "Docker buildx not available. Install Docker Desktop."
fi
# Setup buildx builder
step "Setting up buildx builder"
BUILDER_NAME="fromchat-builder"
BUILDER_EXISTS=false
if docker buildx inspect "$BUILDER_NAME" > /dev/null 2>&1; then
BUILDER_EXISTS=true
if ! docker buildx use "$BUILDER_NAME" > /dev/null 2>&1; then
substep "Recreating builder..."
docker buildx rm "$BUILDER_NAME" > /dev/null 2>&1 || true
BUILDER_EXISTS=false
elif ! docker buildx inspect "$BUILDER_NAME" > /dev/null 2>&1; then
substep "Recreating builder (inspection failed)..."
docker buildx rm "$BUILDER_NAME" > /dev/null 2>&1 || true
BUILDER_EXISTS=false
fi
fi
if [ "$BUILDER_EXISTS" = false ]; then
substep "Creating builder with persistent cache..."
docker buildx create \
--name "$BUILDER_NAME" \
--driver docker-container \
--driver-opt image=moby/buildkit:latest \
--use \
--bootstrap > /dev/null 2>&1
fi
docker buildx use "$BUILDER_NAME" > /dev/null 2>&1
fi
# Detect services
step "Detecting services"
cd "$DEPLOYMENT_DIR"
# Include profile-only services (e.g. caddy) so their images are built and pushed; otherwise prod keeps a stale Caddy image and ignores Caddyfile updates from rsync.
export COMPOSE_PROFILES=production
SERVICES=$(docker compose -f docker-compose.yml config --services 2>/dev/null)
if [ -z "$SERVICES" ]; then
error "No services found in docker-compose.yml"
fi
if ! command -v jq >/dev/null 2>&1; then
error "jq is required for deploy (e.g. brew install jq / sudo apt install jq)"
exit 1
fi
if ! COMPOSE_JSON=$(docker compose -f docker-compose.yml config --format json 2>/dev/null); then
error "docker compose config --format json failed (needs Docker Compose v2.10+)"
exit 1
fi
BUILT_IMAGES=()
PUSHABLE_IMAGES=()
PUSHABLE_SERVICES=()
PUSHABLE_SERVICE_DOCKERFILES=()
PUSHABLE_SERVICE_CONTEXTS=()
PUSHABLE_SERVICE_TARGETS=()
PUSHABLE_IMAGE_INPUT_HASHES=()
for SERVICE in $SERVICES; do
if ! jq -e --arg s "$SERVICE" '(.services[$s].build // false) | type == "object"' <<< "$COMPOSE_JSON" >/dev/null 2>&1; then
continue
fi
IMAGE_TAG="${PROJECT_NAME}-${SERVICE}:latest"
DOCKERFILE_REL=$(jq -r --arg s "$SERVICE" '.services[$s].build.dockerfile // empty' <<< "$COMPOSE_JSON")
CONTEXT_REL=$(jq -r --arg s "$SERVICE" '.services[$s].build.context // empty' <<< "$COMPOSE_JSON")
# Multi-stage deployment/Dockerfile: without --target, the final stage (file_storage) is always tagged.
BUILD_TARGET=$(jq -r --arg s "$SERVICE" '.services[$s].build.target // empty' <<< "$COMPOSE_JSON")
if [ -z "$CONTEXT_REL" ]; then
CONTEXT_REL=".."
fi
if [[ "$CONTEXT_REL" == ".." ]]; then
BUILD_CONTEXT="$PROJECT_ROOT"
elif [[ "$CONTEXT_REL" == /* ]]; then
BUILD_CONTEXT="$CONTEXT_REL"
else
BUILD_CONTEXT="$DEPLOYMENT_DIR/$CONTEXT_REL"
fi
if [ -n "$DOCKERFILE_REL" ]; then
if [[ "$DOCKERFILE_REL" == /* ]]; then
DOCKERFILE="$DOCKERFILE_REL"
else
if [[ "$CONTEXT_REL" == ".." ]] || [[ "$BUILD_CONTEXT" == "$PROJECT_ROOT" ]]; then
DOCKERFILE="$PROJECT_ROOT/$DOCKERFILE_REL"
else
DOCKERFILE="$BUILD_CONTEXT/$DOCKERFILE_REL"
fi
fi
else
if [ -f "$DEPLOYMENT_DIR/Dockerfile.$SERVICE" ]; then
DOCKERFILE="$DEPLOYMENT_DIR/Dockerfile.$SERVICE"
elif [ -f "$DEPLOYMENT_DIR/$SERVICE/Dockerfile" ]; then
DOCKERFILE="$DEPLOYMENT_DIR/$SERVICE/Dockerfile"
else
error "Could not determine Dockerfile for $SERVICE"
fi
fi
if [ ! -f "$INPUT_HASH_TOOL" ]; then
error "Missing $INPUT_HASH_TOOL (needed for dependency hashing)"
exit 1
fi
INPUT_HASH=$(compute_inputs_hash "$BUILD_CONTEXT" "$DOCKERFILE")
PUSHABLE_SERVICES+=("$SERVICE")
PUSHABLE_IMAGES+=("$IMAGE_TAG")
PUSHABLE_SERVICE_DOCKERFILES+=("$DOCKERFILE")
PUSHABLE_SERVICE_CONTEXTS+=("$BUILD_CONTEXT")
PUSHABLE_SERVICE_TARGETS+=("$BUILD_TARGET")
PUSHABLE_IMAGE_INPUT_HASHES+=("$INPUT_HASH")
done
mkdir -p "$LOCAL_IMAGE_CACHE_DIR"
SERVICES_TO_BUILD=()
IMAGES_TO_BUILD=()
INPUT_HASHES_TO_BUILD=()
DOCKERFILES_TO_BUILD=()
CONTEXTS_TO_BUILD=()
TARGETS_TO_BUILD=()
for i in "${!PUSHABLE_SERVICES[@]}"; do
SERVICE="${PUSHABLE_SERVICES[$i]}"
IMAGE_TAG="${PUSHABLE_IMAGES[$i]}"
INPUT_HASH="${PUSHABLE_IMAGE_INPUT_HASHES[$i]}"
DOCKERFILE="${PUSHABLE_SERVICE_DOCKERFILES[$i]}"
BUILD_CONTEXT="${PUSHABLE_SERVICE_CONTEXTS[$i]}"
BUILD_TARGET="${PUSHABLE_SERVICE_TARGETS[$i]}"
CACHE_KEY="$(sanitize_ref "$IMAGE_TAG")"
CACHE_FILE="$LOCAL_IMAGE_CACHE_DIR/$CACHE_KEY/input.sha256"
PREV_HASH="$(read_file_if_exists "$CACHE_FILE")"
LOCAL_FP="$(get_local_image_layer_fp "$IMAGE_TAG")"
if [ -n "$PREV_HASH" ] && [ "$PREV_HASH" = "$INPUT_HASH" ] && [ -n "$LOCAL_FP" ]; then
continue
fi
SERVICES_TO_BUILD+=("$SERVICE")
IMAGES_TO_BUILD+=("$IMAGE_TAG")
INPUT_HASHES_TO_BUILD+=("$INPUT_HASH")
DOCKERFILES_TO_BUILD+=("$DOCKERFILE")
CONTEXTS_TO_BUILD+=("$BUILD_CONTEXT")
TARGETS_TO_BUILD+=("$BUILD_TARGET")
done
if [ ${#SERVICES_TO_BUILD[@]} -eq 0 ]; then
success "Build skipped (no Docker inputs changed)"
else
step "Building ${#SERVICES_TO_BUILD[@]} service(s)"
if [ "$USE_DOCKER_BUILD" = true ]; then
# Same-arch fast path: build with Docker Compose (requested).
# Ensure the project name is stable so the resulting tags match our expectations.
COMPOSE_PROJECT_NAME="$PROJECT_NAME" docker compose -f docker-compose.yml --profile production build "${SERVICES_TO_BUILD[@]}"
else
for i in "${!SERVICES_TO_BUILD[@]}"; do
SERVICE="${SERVICES_TO_BUILD[$i]}"
IMAGE_TAG="${IMAGES_TO_BUILD[$i]}"
DOCKERFILE="${DOCKERFILES_TO_BUILD[$i]}"
BUILD_CONTEXT="${CONTEXTS_TO_BUILD[$i]}"
BUILD_TARGET="${TARGETS_TO_BUILD[$i]}"
substep "Building ${CYAN}$SERVICE${NC} -> ${CYAN}$IMAGE_TAG${NC}..."
# On macOS Docker Desktop, --load can hang for a long time at "sending tarball".
# Use the docker exporter explicitly to load into the local Docker daemon.
BUILDX_ARGS=(buildx build --platform "$PLATFORM" --file "$DOCKERFILE" --tag "$IMAGE_TAG" --output=type=docker --provenance=false --sbom=false)
if [ -n "$BUILD_TARGET" ]; then
BUILDX_ARGS+=(--target "$BUILD_TARGET")
fi
BUILDX_ARGS+=("$BUILD_CONTEXT")
if ! docker "${BUILDX_ARGS[@]}"; then
error "Build failed for $SERVICE"
exit 1
fi
done
fi
for i in "${!SERVICES_TO_BUILD[@]}"; do
IMAGE_TAG="${IMAGES_TO_BUILD[$i]}"
INPUT_HASH="${INPUT_HASHES_TO_BUILD[$i]}"
CACHE_KEY="$(sanitize_ref "$IMAGE_TAG")"
mkdir -p "$LOCAL_IMAGE_CACHE_DIR/$CACHE_KEY"
printf "%s" "$INPUT_HASH" > "$LOCAL_IMAGE_CACHE_DIR/$CACHE_KEY/input.sha256"
BUILT_IMAGES+=("$IMAGE_TAG")
done
success "Build complete! ${#BUILT_IMAGES[@]} image(s) built"
fi
# ============================================================================
# DEPLOY PHASE
# ============================================================================
echo -e "\n${MAGENTA}${BOLD}🚀 Deploying to ${SERVER}${NC}\n"
# Check docker pussh
if ! docker pussh --help > /dev/null 2>&1; then
error "docker pussh plugin not installed"
echo " Install: npm run install:pussh"
fi
# Detect images based on docker-compose.yml (prefer explicit `image:` entries; fall back to built tags)
cd "$DEPLOYMENT_DIR"
COMPOSE_SERVICES=$(docker compose -f docker-compose.yml config --services 2>/dev/null || true)
PUSH_IMAGES=()
EXTERNAL_IMAGES=()
for S in $COMPOSE_SERVICES; do
IMAGE_FROM_COMPOSE=$(jq -r --arg s "$S" '.services[$s].image // empty' <<< "$COMPOSE_JSON")
HAS_BUILD=$(jq -r --arg s "$S" '(.services[$s].build // empty) | (if . == "" then "" else "yes" end)' <<< "$COMPOSE_JSON")
if [ -n "$IMAGE_FROM_COMPOSE" ]; then
# If it has an explicit image and no build section, it likely won't exist locally (and doesn't need pussh).
if [ -z "$HAS_BUILD" ]; then
EXTERNAL_IMAGES+=("$IMAGE_FROM_COMPOSE")
else
# If a service has both build+image, treat it as a built image (pussh).
PUSH_IMAGES+=("$IMAGE_FROM_COMPOSE")
fi
else
# If service has a build section (we built it above), use the tag pattern used during build
TAG="${PROJECT_NAME}-${S}:latest"
# Only include the tag if the image exists locally (avoid pushing unrelated images)
if docker images --format "{{.Repository}}:{{.Tag}}" | grep -q "^${TAG}$"; then
PUSH_IMAGES+=("$TAG")
fi
fi
done
# Deduplicate while preserving order
if [ ${#PUSH_IMAGES[@]} -gt 0 ]; then
PUSH_IMAGES=($(printf "%s\n" "${PUSH_IMAGES[@]}" | awk '!seen[$0]++'))
fi
if [ ${#EXTERNAL_IMAGES[@]} -gt 0 ]; then
EXTERNAL_IMAGES=($(printf "%s\n" "${EXTERNAL_IMAGES[@]}" | awk '!seen[$0]++'))
fi
# Verify that all built images are among the detected images to be pushed.
# This prevents accidentally pushing unrelated images.
BUILT_COUNT=${#BUILT_IMAGES[@]}
MATCHING_BUILT=0
MISSING_FROM_DETECTED=()
# Bash 3.2 + set -u: "${arr[@]}" on an empty array can error; guard with length.
if ((${#BUILT_IMAGES[@]} > 0)); then
for BI in "${BUILT_IMAGES[@]}"; do
found=false
for DI in "${PUSH_IMAGES[@]}"; do
if [ "$BI" = "$DI" ]; then
found=true
break
fi
done
if [ "$found" = true ]; then
MATCHING_BUILT=$((MATCHING_BUILT + 1))
else
MISSING_FROM_DETECTED+=("$BI")
fi
done
fi
# Also list push images that weren't built locally (these are likely prebuilt local images)
NOT_BUILT_DETECTED=()
for DI in "${PUSH_IMAGES[@]}"; do
built=false
if ((${#BUILT_IMAGES[@]} > 0)); then
for BI in "${BUILT_IMAGES[@]}"; do
if [ "$DI" = "$BI" ]; then
built=true
break
fi
done
fi
if [ "$built" = false ]; then
NOT_BUILT_DETECTED+=("$DI")
fi
done
if [ "$BUILT_COUNT" -ne "$MATCHING_BUILT" ]; then
error "Mismatch between built images (${BUILT_COUNT}) and detected built images (${MATCHING_BUILT})."
if [ ${#MISSING_FROM_DETECTED[@]} -gt 0 ]; then
echo " Built but not detected: ${MISSING_FROM_DETECTED[*]}"
fi
if [ ${#NOT_BUILT_DETECTED[@]} -gt 0 ]; then
echo " Detected but not built (external images): ${NOT_BUILT_DETECTED[*]}"
fi
echo "Aborting to avoid pushing incorrect images."
exit 1
fi
if [ ${#PUSH_IMAGES[@]} -eq 0 ] && [ ${#EXTERNAL_IMAGES[@]} -eq 0 ]; then
error "No images found in docker-compose.yml or built locally for project ${PROJECT_NAME}"
fi
# Transfer images — only what was rebuilt this run (BUILT_IMAGES). No remote hash/fingerprint checks.
step "Transferring images"
IMAGES_TO_PUSH=()
if ((${#BUILT_IMAGES[@]} > 0)); then
for PI in "${PUSH_IMAGES[@]}"; do
for BI in "${BUILT_IMAGES[@]}"; do
if [ "$PI" = "$BI" ]; then
IMAGES_TO_PUSH+=("$PI")
break
fi
done
done
fi
if [ ${#IMAGES_TO_PUSH[@]} -eq 0 ]; then
success "Skipping image push (nothing was rebuilt this run)"
else
UNREGISTRY_IMAGE="ghcr.io/psviderski/unregistry"
if ! ssh "$SERVER" "sudo docker images --format '{{.Repository}}:{{.Tag}}' | grep -q '^${UNREGISTRY_IMAGE}$'" 2>/dev/null; then
substep "Pulling unregistry image (one-time setup)..."
if [ -n "${SUDO_PASSWORD:-}" ]; then
ssh "$SERVER" "echo '$SUDO_PASSWORD' | sudo -S -p '' docker pull ${UNREGISTRY_IMAGE}"
else
ssh "$SERVER" "sudo docker pull ${UNREGISTRY_IMAGE}"
fi
fi
for IMAGE in "${IMAGES_TO_PUSH[@]}"; do
substep "Pushing ${CYAN}$IMAGE${NC}..."
if ! docker pussh "$IMAGE" "$SERVER"; then
echo -e " ${RED}✗${NC} Failed to push ${CYAN}$IMAGE${NC}"
echo ""
exit 1
fi
echo ""
done
fi
# Pull external images directly on the server (no pussh)
if [ ${#EXTERNAL_IMAGES[@]} -gt 0 ]; then
step "Pulling external images on server"
for IMAGE in "${EXTERNAL_IMAGES[@]}"; do
substep "Pulling ${CYAN}$IMAGE${NC}..."
QIMG=$(printf '%q' "$IMAGE")
if [ -n "${SUDO_PASSWORD:-}" ]; then
if ssh "$SERVER" "echo '$SUDO_PASSWORD' | sudo -S -p '' docker pull $QIMG"; then
echo ""
else
echo -e " ${RED}✗${NC} Failed to pull ${CYAN}$IMAGE${NC} on server"
echo ""
exit 1
fi
elif ssh "$SERVER" "sudo docker pull $QIMG"; then
echo ""
else
echo -e " ${RED}✗${NC} Failed to pull ${CYAN}$IMAGE${NC} on server"
echo ""
exit 1
fi
done
fi
# Transfer files
step "Transferring deployment files"
# Ensure destination directory exists with proper permissions
if [ -n "$SUDO_PASSWORD" ]; then
ssh "$SERVER" bash << REMOTE_SUDO_SCRIPT > /dev/null 2>&1
set -e
echo '$SUDO_PASSWORD' | sudo -S -p '' mkdir -p $DEPLOY_PATH/deployment $DEPLOY_PATH/backend 2>/dev/null || true
echo '$SUDO_PASSWORD' | sudo -S -p '' chown -R \$(whoami):\$(whoami) $DEPLOY_PATH/deployment $DEPLOY_PATH/backend 2>/dev/null || true
REMOTE_SUDO_SCRIPT
else
ssh "$SERVER" "sudo mkdir -p $DEPLOY_PATH/deployment $DEPLOY_PATH/backend && sudo chown -R \$(whoami):\$(whoami) $DEPLOY_PATH/deployment $DEPLOY_PATH/backend" > /dev/null 2>&1 || true
fi
# Copy deployment directory excluding gitignored files
cd "$PROJECT_ROOT"
substep "Copying deployment directory..."
# Generate exclude file for rsync using git ls-files to list ignored files
EXCLUDE_FILE="/tmp/fromchat-rsync-exclude-$$"
RSYNC_ERROR="/tmp/fromchat-rsync-error-$$"
# Get ignored files in deployment directory and convert to rsync exclude patterns
git ls-files --others --ignored --exclude-standard deployment/ 2>/dev/null | \
sed 's|^deployment/||' > "$EXCLUDE_FILE" || true
# Use rsync with native --exclude-from option
if rsync -avz --delete --exclude-from="$EXCLUDE_FILE" \
"$DEPLOYMENT_DIR/" \
"$SERVER:$DEPLOY_PATH/deployment/" > "$RSYNC_ERROR" 2>&1; then
rm -f "$EXCLUDE_FILE" "$RSYNC_ERROR"
else
echo -e " ${RED}✗${NC} Rsync failed. Error output:"
cat "$RSYNC_ERROR" | sed 's/^/ /'
rm -f "$EXCLUDE_FILE" "$RSYNC_ERROR"
echo -n " " && error "Failed to copy deployment directory"
fi
# Copy .env.prod to .env on server (bypassing gitignore)
if [ -f "$DEPLOYMENT_DIR/.env.prod" ]; then
substep "Copying .env.prod to .env..."
if ! scp "$DEPLOYMENT_DIR/.env.prod" "$SERVER:$DEPLOY_PATH/deployment/.env" > /dev/null 2>&1; then
warning "Failed to copy .env.prod to .env"
fi
else
warning ".env.prod not found in deployment directory"
fi
# Firebase service account: bind-mounted at runtime (see docker-compose main volumes), never in the image (.dockerignore).
# If compose ever ran without this file on the host, Docker may have created a directory at this path — remove it before scp.
FIREBASE_CERT="$PROJECT_ROOT/backend/firebase-cert.json"
substep "Firebase service account (runtime bind-mount: backend/firebase-cert.json)..."
# ~ is not expanded inside variables on the remote shell (e.g. C="$D/..." with D=~/foo checks a bogus path). Resolve to an absolute path.
DEPLOY_PATH_ON_SERVER=$(ssh -o BatchMode=yes -o ConnectTimeout=10 "$SERVER" "eval echo $DEPLOY_PATH" 2>/dev/null || true)
if [ -z "$DEPLOY_PATH_ON_SERVER" ]; then
DEPLOY_PATH_ON_SERVER=$DEPLOY_PATH
fi
FIREBASE_REMOTE="$DEPLOY_PATH_ON_SERVER/backend/firebase-cert.json"
# Docker may have created this path as a root-owned directory; plain rm fails without sudo.
if [ -n "$SUDO_PASSWORD" ]; then
ssh "$SERVER" bash << REMOTE_FIREBASE_CLEANUP > /dev/null 2>&1 || true
set -e
D=$DEPLOY_PATH_ON_SERVER
C="\$D/backend/firebase-cert.json"
mkdir -p "\$D/backend" 2>/dev/null || true
if [ -d "\$C" ]; then
echo '$SUDO_PASSWORD' | sudo -S -p '' rm -rf "\$C"
fi
echo '$SUDO_PASSWORD' | sudo -S -p '' chown -R "\$(whoami):\$(whoami)" "\$D/backend" 2>/dev/null || true
REMOTE_FIREBASE_CLEANUP
else
QBASE=$(printf '%q' "$DEPLOY_PATH_ON_SERVER")
ssh "$SERVER" "D=$QBASE; C=\"\$D/backend/firebase-cert.json\"; mkdir -p \"\$D/backend\"; if [ -d \"\$C\" ]; then sudo rm -rf \"\$C\" 2>/dev/null || rm -rf \"\$C\"; fi; sudo chown -R \$(whoami):\$(whoami) \"\$D/backend\" 2>/dev/null || true" > /dev/null 2>&1 || true
fi
while true; do
if [ -f "$FIREBASE_CERT" ]; then
break
fi
if [ -d "$FIREBASE_CERT" ]; then
echo -e " ${YELLOW}⚠${NC} $FIREBASE_CERT is a directory. Delete it and save the Firebase service account JSON as a file at that exact path."
elif [ -e "$FIREBASE_CERT" ]; then
echo -e " ${YELLOW}⚠${NC} $FIREBASE_CERT exists but is not a regular file."
else
echo -e " ${YELLOW}⚠${NC} Missing $FIREBASE_CERT (Firebase service account JSON for FCM)."
fi
echo -e " ${CYAN}Fix this, then press Enter to check again (Ctrl+C to abort deploy).${NC}"
read -r _
done
substep "Copying backend/firebase-cert.json..."
FIREBASE_SCP_LOG="/tmp/fromchat-firebase-scp-$$.log"
if ! scp "$FIREBASE_CERT" "$SERVER:$FIREBASE_REMOTE" >"$FIREBASE_SCP_LOG" 2>&1; then
error "Failed to copy firebase-cert.json to server"
echo -e " ${YELLOW}Target:${NC} $SERVER:$FIREBASE_REMOTE" >&2
if [ -s "$FIREBASE_SCP_LOG" ]; then
sed 's/^/ /' "$FIREBASE_SCP_LOG" >&2
else
echo -e " ${YELLOW}(scp produced no output.)${NC}" >&2
fi
rm -f "$FIREBASE_SCP_LOG"
exit 1
fi
rm -f "$FIREBASE_SCP_LOG"
ssh "$SERVER" "chmod 600 $(printf '%q' "$FIREBASE_REMOTE") 2>/dev/null || true"
if ! ssh "$SERVER" "test -f $(printf '%q' "$FIREBASE_REMOTE")"; then
error "Server path is not a regular file after copy: $FIREBASE_REMOTE"
exit 1
fi
# Deploy on server
step "Deploying on server"
ssh "$SERVER" SUDO_PASSWORD="$SUDO_PASSWORD" DEPLOY_PATH="${DEPLOY_PATH_ON_SERVER:-$DEPLOY_PATH}" bash << 'REMOTE_SCRIPT'
set -e
REMOTE_SUDO_PASS="${SUDO_PASSWORD:-}"
REMOTE_DEPLOY_PATH="${DEPLOY_PATH:-}"
export SUDO_PROMPT=""
sudo_cmd() {
if [ -n "$REMOTE_SUDO_PASS" ]; then
echo "$REMOTE_SUDO_PASS" | sudo -S -p '' "$@" 2>/dev/null
else
sudo "$@" 2>/dev/null
fi
}
if [ -z "$REMOTE_DEPLOY_PATH" ]; then
echo "❌ DEPLOY_PATH is not set"
exit 1
fi
mkdir -p "$REMOTE_DEPLOY_PATH/deployment" "$REMOTE_DEPLOY_PATH/backend"
cd "$REMOTE_DEPLOY_PATH/deployment"
if [ ! -f "$REMOTE_DEPLOY_PATH/deployment/.env" ]; then
echo "⚠️ Warning: .env file not found"
fi
if systemctl is-active --quiet fromchat; then
sudo_cmd systemctl stop fromchat
fi
COMPOSE_PROFILES=production docker compose down --remove-orphans > /dev/null 2>&1 || true
sudo_cmd cp -f "$REMOTE_DEPLOY_PATH/deployment/fromchat.service" /etc/systemd/system/fromchat.service
sudo_cmd systemctl daemon-reload
sudo_cmd systemctl restart fromchat
sleep 3
if ! systemctl is-active --quiet fromchat; then
echo "❌ Service failed to start"
sudo_cmd journalctl --no-pager -xeu fromchat -n 30
exit 1
fi
REMOTE_SCRIPT
echo
success "Deployment complete!"
scripts/deploy/__init__.py
+1
View File
@@ -0,0 +1 @@
"""FromChat deployment orchestration (Docker build, pussh, rsync, remote systemd)."""
scripts/deploy/compose_build.py
+272
View File
@@ -0,0 +1,272 @@
"""Parse docker-compose JSON and run image builds."""
from __future__ import annotations
import json
import os
import subprocess
import sys
from dataclasses import dataclass
from pathlib import Path
from deploy.paths import ProjectPaths
import deploy.ui as ui
from deploy.util import (
compute_inputs_hash,
dedupe_preserve,
local_image_layer_fp,
read_file_if_exists,
sanitize_ref,
)
def remote_project_name(server: str, deploy_path: str) -> str:
r = subprocess.run(
["ssh", server, f"dirname {deploy_path}/deployment/docker-compose.yml"],
capture_output=True,
text=True,
)
compose_dir = r.stdout.strip() if r.returncode == 0 and r.stdout.strip() else f"{deploy_path}/deployment"
r2 = subprocess.run(["ssh", server, f"basename {compose_dir}"], capture_output=True, text=True)
if r2.returncode == 0 and r2.stdout.strip():
return r2.stdout.strip()
return "deployment"
@dataclass
class PushableService:
service: str
image_tag: str
dockerfile: Path
build_context: Path
build_target: str
input_hash: str
class ComposeBuildPhase:
def __init__(
self,
paths: ProjectPaths,
*,
project_name: str,
platform: str,
use_docker_build: bool,
) -> None:
self._paths = paths
self._project_name = project_name
self._platform = platform
self._use_docker_build = use_docker_build
def load_compose_json(self, deployment_dir: Path) -> dict:
env = os.environ.copy()
env["COMPOSE_PROFILES"] = "production"
p = subprocess.run(
["docker", "compose", "-f", "docker-compose.yml", "config", "--format", "json"],
cwd=deployment_dir,
capture_output=True,
text=True,
env=env,
)
if p.returncode != 0:
ui.error("docker compose config --format json failed (needs Docker Compose v2.10+)")
sys.exit(1)
return json.loads(p.stdout)
def list_services(self, deployment_dir: Path) -> list[str]:
env = os.environ.copy()
env["COMPOSE_PROFILES"] = "production"
p = subprocess.run(
["docker", "compose", "-f", "docker-compose.yml", "config", "--services"],
cwd=deployment_dir,
capture_output=True,
text=True,
env=env,
)
if p.returncode != 0:
return []
return [s.strip() for s in p.stdout.splitlines() if s.strip()]
def collect_pushable(self, compose: dict, services: list[str]) -> list[PushableService]:
deployment_dir = self._paths.deployment_dir
project_root = self._paths.project_root
out: list[PushableService] = []
svc_map = compose.get("services") or {}
for service in services:
spec = svc_map.get(service)
if not isinstance(spec, dict):
continue
build = spec.get("build")
if not isinstance(build, dict):
continue
image_tag = f"{self._project_name}-{service}:latest"
dockerfile_rel = (build.get("dockerfile") or "").strip()
context_rel = (build.get("context") or "").strip()
build_target = (build.get("target") or "").strip()
if not context_rel:
context_rel = ".."
if context_rel == "..":
build_context = project_root
elif context_rel.startswith("/"):
build_context = Path(context_rel)
else:
build_context = deployment_dir / context_rel
if dockerfile_rel:
if dockerfile_rel.startswith("/"):
dockerfile = Path(dockerfile_rel)
elif context_rel == ".." or build_context == project_root:
dockerfile = project_root / dockerfile_rel
else:
dockerfile = build_context / dockerfile_rel
else:
cand_a = deployment_dir / f"Dockerfile.{service}"
cand_b = deployment_dir / service / "Dockerfile"
if cand_a.is_file():
dockerfile = cand_a
elif cand_b.is_file():
dockerfile = cand_b
else:
ui.error(f"Could not determine Dockerfile for {service}")
sys.exit(1)
if not self._paths.input_hash_script.is_file():
ui.error(f"Missing {self._paths.input_hash_script} (needed for dependency hashing)")
sys.exit(1)
h = compute_inputs_hash(
build_context,
dockerfile,
hash_script=self._paths.input_hash_script,
)
if not h:
ui.error(f"Failed to compute input hash for {service}")
sys.exit(1)
out.append(
PushableService(
service=service,
image_tag=image_tag,
dockerfile=dockerfile,
build_context=build_context,
build_target=build_target,
input_hash=h,
)
)
return out
def plan_builds(self, pushable: list[PushableService]) -> tuple[list[PushableService], list[str]]:
"""Return (to_build, built_images_after) — built_images empty until build runs."""
cache_root = self._paths.local_image_cache_dir
cache_root.mkdir(parents=True, exist_ok=True)
to_build: list[PushableService] = []
for ps in pushable:
key = sanitize_ref(ps.image_tag)
cache_file = cache_root / key / "input.sha256"
prev = read_file_if_exists(cache_file).strip()
fp = local_image_layer_fp(ps.image_tag)
if prev and prev == ps.input_hash and fp:
continue
to_build.append(ps)
return to_build, []
def run_builds(self, to_build: list[PushableService]) -> list[str]:
if not to_build:
ui.success("Build skipped (no Docker inputs changed)")
return []
ui.step(f"Building {len(to_build)} service(s)")
deployment_dir = self._paths.deployment_dir
env = os.environ.copy()
env["COMPOSE_PROJECT_NAME"] = self._project_name
env["COMPOSE_PROFILES"] = "production"
if self._use_docker_build:
cmd = [
"docker",
"compose",
"-f",
"docker-compose.yml",
"--profile",
"production",
"build",
*[p.service for p in to_build],
]
if subprocess.run(cmd, cwd=deployment_dir, env=env).returncode != 0:
ui.error("docker compose build failed")
sys.exit(1)
else:
for ps in to_build:
ui.substep(f"Building {ps.service} -> {ps.image_tag}...")
args = [
"docker",
"buildx",
"build",
"--platform",
self._platform,
"--file",
str(ps.dockerfile),
"--tag",
ps.image_tag,
"--output=type=docker",
"--provenance=false",
"--sbom=false",
]
if ps.build_target:
args.extend(["--target", ps.build_target])
args.append(str(ps.build_context))
if subprocess.run(args).returncode != 0:
ui.error(f"Build failed for {ps.service}")
sys.exit(1)
built: list[str] = []
for ps in to_build:
key = sanitize_ref(ps.image_tag)
d = self._paths.local_image_cache_dir / key
d.mkdir(parents=True, exist_ok=True)
(d / "input.sha256").write_text(ps.input_hash, encoding="utf-8")
built.append(ps.image_tag)
ui.success(f"Build complete! {len(built)} image(s) built")
return built
def classify_push_and_external(
compose: dict,
project_name: str,
local_tags: set[str],
service_order: list[str],
) -> tuple[list[str], list[str]]:
services = compose.get("services") or {}
push_images: list[str] = []
external: list[str] = []
for name in service_order:
spec = services.get(name)
if not isinstance(spec, dict):
continue
image_from = (spec.get("image") or "").strip()
build = spec.get("build")
has_build = isinstance(build, dict)
if image_from:
if not has_build:
external.append(image_from)
else:
push_images.append(image_from)
else:
tag = f"{project_name}-{name}:latest"
if tag in local_tags:
push_images.append(tag)
return dedupe_preserve(push_images), dedupe_preserve(external)
def verify_built_subset_push(built: list[str], push_images: list[str], ui: object) -> None:
matching = sum(1 for bi in built if bi in push_images)
missing = [bi for bi in built if bi not in push_images]
not_built = [di for di in push_images if di not in built]
if len(built) != matching:
ui.error(f"Mismatch between built images ({len(built)}) and detected built images ({matching}).")
if missing:
print(f" Built but not detected: {' '.join(missing)}")
if not_built:
print(f" Detected but not built (external images): {' '.join(not_built)}")
print("Aborting to avoid pushing incorrect images.")
sys.exit(1)
def images_to_push_intersection(push_images: list[str], built: list[str]) -> list[str]:
out: list[str] = []
for pi in push_images:
if pi in built:
out.append(pi)
return dedupe_preserve(out)
scripts/deploy/config.py
+70
View File
@@ -0,0 +1,70 @@
"""Load deployment/.env and CLI into settings."""
from __future__ import annotations
import os
import platform
import sys
from dataclasses import dataclass
from pathlib import Path
from dotenv import load_dotenv
from deploy.paths import ProjectPaths
@dataclass
class DeploySettings:
server: str
repo_name: str
deploy_path: str
platform: str
host_arch: str
platform_arch: str
use_docker_build: bool
paths: ProjectPaths
def _machine_arch() -> str:
m = platform.machine().lower()
if m in ("arm64", "aarch64"):
return "arm64"
if m in ("x86_64", "amd64", "i386", "i686"):
return "amd64"
return m
def load_settings(paths: ProjectPaths, argv: list[str]) -> DeploySettings:
if paths.env_file.is_file():
load_dotenv(paths.env_file, override=False)
server = (argv[1] if len(argv) > 1 else None) or os.environ.get("DEPLOYMENT_SERVER", "")
server = server.strip()
if not server:
sys.stderr.write(
"Server not specified. Usage: deploy.sh [user@host] [deployment_path] [platform]\n"
f" Or set DEPLOYMENT_SERVER in {paths.env_file} or as an environment variable\n\n"
"Example:\n"
" deploy.sh user@example.com /home/user/fromchat linux/arm64\n"
f" Or add to {paths.env_file}: DEPLOYMENT_SERVER=user@example.com\n"
)
raise SystemExit(1)
repo_name = "FromChat"
deploy_path = f"~/actions-runner/_work/{repo_name}/{repo_name}"
docker_platform = "linux/arm64"
host_arch = _machine_arch()
platform_arch = docker_platform.split("/", 1)[-1]
use_docker_build = bool(host_arch and host_arch == platform_arch)
return DeploySettings(
server=server,
repo_name=repo_name,
deploy_path=deploy_path,
platform=docker_platform,
host_arch=host_arch,
platform_arch=platform_arch,
use_docker_build=use_docker_build,
paths=paths,
)
scripts/deploy/docker_local.py
+87
View File
@@ -0,0 +1,87 @@
"""Local Docker daemon, Docker Desktop, and buildx setup."""
from __future__ import annotations
import subprocess
import sys
import time
import deploy.ui as ui
BUILDER_NAME = "fromchat-builder"
def ensure_daemon() -> None:
if _daemon_ok():
return
ui.warning("Docker daemon is not running")
if not _start_desktop():
ui.error("Failed to start Docker Desktop. Please start it manually and try again.")
sys.exit(1)
def _daemon_ok() -> bool:
return subprocess.run(["docker", "info"], capture_output=True).returncode == 0
def _start_desktop() -> bool:
ui.substep("Starting Docker Desktop...")
if subprocess.run(["docker", "desktop", "start"], capture_output=True).returncode != 0:
return False
ui.substep("Waiting for Docker to start...", end="")
sys.stdout.flush()
max_wait = 60
waited = 0
while waited < max_wait:
if _daemon_ok():
print()
return True
time.sleep(2)
waited += 2
print(".", end="", flush=True)
print()
return False
def ensure_buildx(use_compose_build: bool) -> None:
if use_compose_build:
return
if subprocess.run(["docker", "buildx", "version"], capture_output=True).returncode != 0:
ui.error("Docker buildx not available. Install Docker Desktop.")
sys.exit(1)
_setup_builder()
def _setup_builder() -> None:
ui.step("Setting up buildx builder")
name = BUILDER_NAME
exists = subprocess.run(["docker", "buildx", "inspect", name], capture_output=True).returncode == 0
if exists:
if subprocess.run(["docker", "buildx", "use", name], capture_output=True).returncode != 0:
ui.substep("Recreating builder...")
subprocess.run(["docker", "buildx", "rm", name], capture_output=True)
exists = False
elif subprocess.run(["docker", "buildx", "inspect", name], capture_output=True).returncode != 0:
ui.substep("Recreating builder (inspection failed)...")
subprocess.run(["docker", "buildx", "rm", name], capture_output=True)
exists = False
if not exists:
ui.substep("Creating builder with persistent cache...")
subprocess.run(
[
"docker",
"buildx",
"create",
"--name",
name,
"--driver",
"docker-container",
"--driver-opt",
"image=moby/buildkit:latest",
"--use",
"--bootstrap",
],
capture_output=True,
)
subprocess.run(["docker", "buildx", "use", name], capture_output=True)
scripts/deploy/main.py
+90
View File
@@ -0,0 +1,90 @@
"""CLI entry: build Docker images, pussh, rsync, restart remote systemd."""
from __future__ import annotations
import sys
from pathlib import Path
_SCRIPTS = Path(__file__).resolve().parent.parent
if str(_SCRIPTS) not in sys.path:
sys.path.insert(0, str(_SCRIPTS))
from deploy.compose_build import ( # noqa: E402
ComposeBuildPhase,
classify_push_and_external,
images_to_push_intersection,
remote_project_name,
verify_built_subset_push,
)
import deploy.ui as ui # noqa: E402
from deploy.config import load_settings # noqa: E402
import deploy.docker_local as docker_local # noqa: E402
from deploy.paths import ProjectPaths # noqa: E402
from deploy.ssh_auth import SshAuth # noqa: E402
from deploy.transfer import DeployTransfer # noqa: E402
from deploy.util import local_docker_image_tags # noqa: E402
def main() -> None:
paths = ProjectPaths.from_deploy_package()
settings = load_settings(paths, sys.argv)
ui.banner()
creds = SshAuth(settings.server).authenticate()
project_name = remote_project_name(settings.server, settings.deploy_path)
ui.build_banner()
docker_local.ensure_daemon()
docker_local.ensure_buildx(settings.use_docker_build)
ui.step("Detecting services")
build_phase = ComposeBuildPhase(
paths,
project_name=project_name,
platform=settings.platform,
use_docker_build=settings.use_docker_build,
)
deployment_dir = paths.deployment_dir
services = build_phase.list_services(deployment_dir)
if not services:
ui.error("No services found in docker-compose.yml")
raise SystemExit(1)
compose_json = build_phase.load_compose_json(deployment_dir)
pushable = build_phase.collect_pushable(compose_json, services)
to_build, _ = build_phase.plan_builds(pushable)
built_images = build_phase.run_builds(to_build)
ui.deploy_banner(settings.server)
transfer = DeployTransfer(paths)
transfer.ensure_pussh()
push_images, external_images = classify_push_and_external(
compose_json,
project_name,
local_docker_image_tags(),
services,
)
verify_built_subset_push(built_images, push_images, ui)
if not push_images and not external_images:
ui.error(f"No images found in docker-compose.yml or built locally for project {project_name}")
raise SystemExit(1)
to_push = images_to_push_intersection(push_images, built_images)
transfer.pussh_images(creds, to_push)
transfer.pull_external_on_server(creds, external_images)
transfer.rsync_deployment(creds, settings.deploy_path)
transfer.copy_env_prod(creds, settings.deploy_path)
deploy_resolved = transfer.sync_firebase_cert(creds, settings.deploy_path)
transfer.run_remote_systemd(creds, deploy_resolved)
print()
ui.success("Deployment complete!")
if __name__ == "__main__":
main()
scripts/deploy/paths.py
+35
View File
@@ -0,0 +1,35 @@
"""Resolved filesystem paths for the Web repo."""
from __future__ import annotations
from dataclasses import dataclass
from pathlib import Path
@dataclass(frozen=True)
class ProjectPaths:
"""Root and well-known directories (Web repo root = project root)."""
project_root: Path
scripts_dir: Path
deployment_dir: Path
env_file: Path
local_cache_root: Path
local_image_cache_dir: Path
input_hash_script: Path
@classmethod
def from_deploy_package(cls) -> ProjectPaths:
deploy_dir = Path(__file__).resolve().parent
scripts_dir = deploy_dir.parent
project_root = scripts_dir.parent
deployment_dir = project_root / "deployment"
return cls(
project_root=project_root,
scripts_dir=scripts_dir,
deployment_dir=deployment_dir,
env_file=deployment_dir / ".env",
local_cache_root=project_root / ".deploy-cache",
local_image_cache_dir=project_root / ".deploy-cache" / "images",
input_hash_script=scripts_dir / "docker_inputs_hash.py",
)
scripts/deploy/ssh_auth.py
+109
View File
@@ -0,0 +1,109 @@
"""SSH key agent and optional sudo password for remote."""
from __future__ import annotations
import getpass
import os
import subprocess
import sys
from dataclasses import dataclass
from pathlib import Path
import deploy.ui as ui
@dataclass
class SshCredentials:
server: str
sudo_password: str
class SshAuth:
def __init__(self, server: str) -> None:
self._server = server
def authenticate(self) -> SshCredentials:
ui.step("Authentication")
self._ensure_agent()
key_file = Path.home() / ".ssh" / "id_rsa"
self._ensure_key_file(key_file)
self._ensure_key_in_agent(key_file)
self._verify_key_auth(key_file)
sudo_password = self._prompt_sudo()
return SshCredentials(server=self._server, sudo_password=sudo_password)
def _ensure_agent(self) -> None:
if os.environ.get("SSH_AUTH_SOCK"):
return
subprocess.run(["ssh-agent", "-s"], capture_output=True, check=False)
def _ensure_key_file(self, key_file: Path) -> None:
if not key_file.is_file():
ui.error(f"SSH key not found at {key_file}")
sys.stderr.write(
" Please generate an SSH key pair first:\n"
" ssh-keygen -t rsa -b 4096 -C 'your_email@example.com'\n"
)
raise SystemExit(1)
def _ensure_key_in_agent(self, key_file: Path) -> None:
loaded = False
r = subprocess.run(["ssh-add", "-l"], capture_output=True, text=True)
if r.returncode == 0:
fp_r = subprocess.run(
["ssh-keygen", "-lf", str(key_file)],
capture_output=True,
text=True,
)
if fp_r.returncode == 0:
parts = fp_r.stdout.strip().split()
fingerprint = parts[1] if len(parts) > 1 else ""
if fingerprint and fingerprint in r.stdout:
loaded = True
if not loaded:
ui.substep("Adding SSH key to agent...")
if subprocess.run(["ssh-add", str(key_file)], capture_output=True).returncode != 0:
ui.error("Failed to add SSH key to agent. Check your key passphrase.")
raise SystemExit(1)
def _verify_key_auth(self, key_file: Path) -> None:
pub = key_file.with_suffix(key_file.suffix + ".pub")
ok = subprocess.run(
[
"ssh",
"-o",
"BatchMode=yes",
"-o",
"ConnectTimeout=10",
"-o",
"StrictHostKeyChecking=no",
self._server,
"echo 'SSH key works'",
],
capture_output=True,
).returncode
if ok == 0:
return
ui.error(f"SSH key authentication failed for {self._server}")
sys.stderr.write(
f' Copy your public key to the server, then re-run deploy:\n ssh-copy-id -i "{pub}" "{self._server}"\n\n'
" Or manually append this key to ~/.ssh/authorized_keys on the server:\n"
)
if pub.is_file():
sys.stderr.write(f" {pub.read_text(encoding='utf-8', errors='replace').strip()}\n")
raise SystemExit(1)
def _prompt_sudo(self) -> str:
while True:
pw = getpass.getpass(" • Sudo password: ")
if not pw:
ui.warning("No password provided - assuming passwordless sudo")
return ""
chk = subprocess.run(
["ssh", self._server, "sudo", "-S", "-v"],
input=(pw + "\n").encode(),
capture_output=True,
)
if chk.returncode == 0:
return pw
ui.error("Invalid password, please try again")
scripts/deploy/transfer.py
+288
View File
@@ -0,0 +1,288 @@
"""Image pussh, rsync deployment, Firebase cert, remote systemd."""
from __future__ import annotations
import shlex
import subprocess
import sys
import tempfile
from pathlib import Path
from deploy.paths import ProjectPaths
from deploy.ssh_auth import SshCredentials
import deploy.ui as ui
UNREGISTRY_IMAGE = "ghcr.io/psviderski/unregistry"
REMOTE_SYSTEMD_SCRIPT = r"""set -e
REMOTE_SUDO_PASS="${SUDO_PASSWORD:-}"
REMOTE_DEPLOY_PATH="${DEPLOY_PATH:-}"
export SUDO_PROMPT=""
sudo_cmd() {
if [ -n "$REMOTE_SUDO_PASS" ]; then
echo "$REMOTE_SUDO_PASS" | sudo -S -p '' "$@" 2>/dev/null
else
sudo "$@" 2>/dev/null
fi
}
if [ -z "$REMOTE_DEPLOY_PATH" ]; then
echo "❌ DEPLOY_PATH is not set"
exit 1
fi
mkdir -p "$REMOTE_DEPLOY_PATH/deployment" "$REMOTE_DEPLOY_PATH/backend"
cd "$REMOTE_DEPLOY_PATH/deployment"
if [ ! -f "$REMOTE_DEPLOY_PATH/deployment/.env" ]; then
echo "⚠️ Warning: .env file not found"
fi
if systemctl is-active --quiet fromchat; then
sudo_cmd systemctl stop fromchat
fi
COMPOSE_PROFILES=production docker compose down --remove-orphans > /dev/null 2>&1 || true
sudo_cmd cp -f "$REMOTE_DEPLOY_PATH/deployment/fromchat.service" /etc/systemd/system/fromchat.service
sudo_cmd systemctl daemon-reload
sudo_cmd systemctl restart fromchat
sleep 3
if ! systemctl is-active --quiet fromchat; then
echo "❌ Service failed to start"
sudo_cmd journalctl --no-pager -xeu fromchat -n 30
exit 1
fi
"""
class DeployTransfer:
def __init__(self, paths: ProjectPaths) -> None:
self._paths = paths
def ensure_pussh(self) -> None:
if subprocess.run(["docker", "pussh", "--help"], capture_output=True).returncode != 0:
ui.error("docker pussh plugin not installed")
print(" Install: npm run install:pussh")
def ensure_unregistry(self, creds: SshCredentials) -> None:
check = (
"sudo docker images --format '{{.Repository}}:{{.Tag}}' | "
f"grep -q '^{UNREGISTRY_IMAGE}$'"
)
if subprocess.run(["ssh", creds.server, check], capture_output=True).returncode == 0:
return
ui.substep("Pulling unregistry image (one-time setup)...")
if creds.sudo_password:
inner = f"echo {shlex.quote(creds.sudo_password)} | sudo -S -p '' docker pull {UNREGISTRY_IMAGE}"
else:
inner = f"sudo docker pull {UNREGISTRY_IMAGE}"
subprocess.run(["ssh", creds.server, inner])
def pussh_images(self, creds: SshCredentials, images: list[str]) -> None:
ui.step("Transferring images")
if not images:
ui.success("Skipping image push (nothing was rebuilt this run)")
return
self.ensure_unregistry(creds)
for image in images:
ui.substep(f"Pushing {image}...")
if subprocess.run(["docker", "pussh", image, creds.server]).returncode != 0:
ui.error(f"Failed to push {image}")
raise SystemExit(1)
print()
def pull_external_on_server(self, creds: SshCredentials, images: list[str]) -> None:
if not images:
return
ui.step("Pulling external images on server")
for image in images:
ui.substep(f"Pulling {image}...")
if creds.sudo_password:
inner = f"echo {shlex.quote(creds.sudo_password)} | sudo -S -p '' docker pull {shlex.quote(image)}"
else:
inner = f"sudo docker pull {shlex.quote(image)}"
if subprocess.run(["ssh", creds.server, inner]).returncode != 0:
ui.error(f"Failed to pull {image} on server")
raise SystemExit(1)
print()
def prepare_remote_dirs(self, creds: SshCredentials, deploy_path: str) -> None:
dp = deploy_path
d_dep = shlex.quote(f"{dp}/deployment")
d_back = shlex.quote(f"{dp}/backend")
if creds.sudo_password:
pw = shlex.quote(creds.sudo_password)
script = f"""set -e
echo {pw} | sudo -S -p '' mkdir -p {d_dep} {d_back} 2>/dev/null || true
echo {pw} | sudo -S -p '' chown -R $(whoami):$(whoami) {d_dep} {d_back} 2>/dev/null || true
"""
subprocess.run(["ssh", creds.server, "bash"], input=script.encode(), capture_output=True)
else:
subprocess.run(
[
"ssh",
creds.server,
f"sudo mkdir -p {d_dep} {d_back} && sudo chown -R $(whoami):$(whoami) {d_dep} {d_back}",
],
capture_output=True,
)
def rsync_deployment(self, creds: SshCredentials, deploy_path: str) -> None:
ui.step("Transferring deployment files")
self.prepare_remote_dirs(creds, deploy_path)
project_root = self._paths.project_root
deployment_dir = self._paths.deployment_dir
ui.substep("Copying deployment directory...")
gl = subprocess.run(
["git", "ls-files", "--others", "--ignored", "--exclude-standard", "deployment/"],
cwd=project_root,
capture_output=True,
text=True,
)
lines = [ln.replace("deployment/", "", 1) for ln in gl.stdout.splitlines() if ln.strip()]
with tempfile.NamedTemporaryFile("w", suffix="-rsync-exclude", delete=False, encoding="utf-8") as tf:
exclude_path = Path(tf.name)
tf.write("\n".join(lines))
try:
rsync = subprocess.run(
[
"rsync",
"-avz",
"--delete",
f"--exclude-from={exclude_path}",
f"{deployment_dir}/",
f"{creds.server}:{deploy_path}/deployment/",
],
cwd=project_root,
capture_output=True,
text=True,
)
if rsync.returncode != 0:
ui.error("Rsync failed. Error output:")
for line in (rsync.stderr or rsync.stdout or "").splitlines():
print(f" {line}")
ui.error("Failed to copy deployment directory")
raise SystemExit(1)
finally:
exclude_path.unlink(missing_ok=True)
def copy_env_prod(self, creds: SshCredentials, deploy_path: str) -> None:
prod = self._paths.deployment_dir / ".env.prod"
if prod.is_file():
ui.substep("Copying .env.prod to .env...")
if subprocess.run(["scp", str(prod), f"{creds.server}:{deploy_path}/deployment/.env"], capture_output=True).returncode != 0:
ui.warning("Failed to copy .env.prod to .env")
else:
ui.warning(".env.prod not found in deployment directory")
def resolve_deploy_path_on_server(self, server: str, deploy_path: str) -> str:
r = subprocess.run(
["ssh", "-o", "BatchMode=yes", "-o", "ConnectTimeout=10", server, f"eval echo {deploy_path}"],
capture_output=True,
text=True,
)
out = r.stdout.strip()
return out if out else deploy_path
def firebase_cert_path(self) -> Path:
return self._paths.project_root / "backend" / "firebase-cert.json"
def cleanup_remote_firebase_dir(self, creds: SshCredentials, deploy_path_resolved: str) -> None:
d = deploy_path_resolved
if creds.sudo_password:
pw = shlex.quote(creds.sudo_password)
script = f"""set -e
D={shlex.quote(d)}
C="$D/backend/firebase-cert.json"
mkdir -p "$D/backend" 2>/dev/null || true
if [ -d "$C" ]; then
echo {pw} | sudo -S -p '' rm -rf "$C"
fi
echo {pw} | sudo -S -p '' chown -R "$(whoami):$(whoami)" "$D/backend" 2>/dev/null || true
"""
subprocess.run(["ssh", creds.server, "bash"], input=script.encode(), capture_output=True)
else:
q = shlex.quote(d)
subprocess.run(
[
"ssh",
creds.server,
f"D={q}; C=\"$D/backend/firebase-cert.json\"; mkdir -p \"$D/backend\"; "
f'if [ -d "$C" ]; then sudo rm -rf "$C" 2>/dev/null || rm -rf "$C"; fi; '
f'sudo chown -R $(whoami):$(whoami) "$D/backend" 2>/dev/null || true',
],
capture_output=True,
)
def _wait_firebase_loop(self, cert: Path) -> None:
while True:
if cert.is_file():
return
if cert.is_dir():
print(
f" ⚠ {cert} is a directory. Delete it and save the Firebase service account JSON as a file at that exact path."
)
elif cert.exists():
print(f" ⚠ {cert} exists but is not a regular file.")
else:
print(f" ⚠ Missing {cert} (Firebase service account JSON for FCM).")
print(" Fix this, then press Enter to check again (Ctrl+C to abort deploy).")
input()
def sync_firebase_cert(self, creds: SshCredentials, deploy_path: str) -> str:
ui.substep(
"Firebase service account (runtime bind-mount: backend/firebase-cert.json)..."
)
resolved = self.resolve_deploy_path_on_server(creds.server, deploy_path)
self.cleanup_remote_firebase_dir(creds, resolved)
cert = self.firebase_cert_path()
self._wait_firebase_loop(cert)
remote = f"{resolved}/backend/firebase-cert.json"
self.scp_firebase(creds, cert, remote)
return resolved
def scp_firebase(self, creds: SshCredentials, cert: Path, remote_path: str) -> None:
ui.substep("Copying backend/firebase-cert.json...")
r = subprocess.run(
["scp", str(cert), f"{creds.server}:{remote_path}"],
capture_output=True,
text=True,
)
if r.returncode != 0:
ui.error("Failed to copy firebase-cert.json to server")
print(f" Target: {creds.server}:{remote_path}", file=sys.stderr)
err = (r.stderr or r.stdout or "").strip()
if err:
for line in err.splitlines():
print(f" {line}", file=sys.stderr)
else:
print(" (scp produced no output.)", file=sys.stderr)
raise SystemExit(1)
subprocess.run(
["ssh", creds.server, f"chmod 600 {shlex.quote(remote_path)}"],
capture_output=True,
)
t = subprocess.run(
["ssh", creds.server, f"test -f {shlex.quote(remote_path)}"],
capture_output=True,
)
if t.returncode != 0:
ui.error(f"Server path is not a regular file after copy: {remote_path}")
raise SystemExit(1)
def run_remote_systemd(self, creds: SshCredentials, deploy_path_resolved: str) -> None:
ui.step("Deploying on server")
pw = creds.sudo_password
dp = deploy_path_resolved
remote_cmd = f"SUDO_PASSWORD={shlex.quote(pw)} DEPLOY_PATH={shlex.quote(dp)} bash -s"
r = subprocess.run(
["ssh", creds.server, remote_cmd],
input=REMOTE_SYSTEMD_SCRIPT.encode(),
text=False,
)
if r.returncode != 0:
raise SystemExit(r.returncode)
scripts/deploy/ui.py
+49
View File
@@ -0,0 +1,49 @@
"""Terminal output (Rich), matching previous deploy.sh style."""
from __future__ import annotations
from rich.console import Console
from rich.text import Text
_console = Console(highlight=False)
def banner() -> None:
_console.print()
_console.print(Text("🚀 Deployment", style="bold magenta"))
_console.print()
def build_banner() -> None:
_console.print()
_console.print(Text("🔨 Building Docker images", style="bold magenta"))
_console.print()
def deploy_banner(server: str) -> None:
_console.print()
_console.print(Text(f"🚀 Deploying to {server}", style="bold magenta"))
_console.print()
def info(msg: str) -> None:
_console.print(Text("ℹ ", style="blue"), msg, sep="")
def success(msg: str) -> None:
_console.print(Text("✓ ", style="green"), msg, sep="")
def warning(msg: str) -> None:
_console.print(Text("⚠ ", style="yellow"), msg, sep="")
def error(msg: str) -> None:
_console.print(Text("✗ ", style="red"), msg, sep="")
def step(msg: str) -> None:
_console.print(Text("→ ", style="bold cyan"), Text(msg, style="bold"), sep="")
def substep(msg: str, *, end: str = "\n") -> None:
_console.print(Text(" • ", style="green"), msg, sep="", end=end)
scripts/deploy/util.py
+85
View File
@@ -0,0 +1,85 @@
"""Small helpers: hashing, dedupe, cache keys."""
from __future__ import annotations
import hashlib
import subprocess
import sys
from pathlib import Path
def sanitize_ref(ref: str) -> str:
s = ref.replace("/", "_").replace(":", "__").replace("@", "__at__")
return s
def dedupe_preserve(items: list[str]) -> list[str]:
seen: set[str] = set()
out: list[str] = []
for x in items:
if x not in seen:
seen.add(x)
out.append(x)
return out
def read_file_if_exists(path: Path) -> str:
if path.is_file():
return path.read_text(encoding="utf-8", errors="replace")
return ""
def local_image_layer_fp(image: str) -> str:
def inspect_layers(ref: str) -> subprocess.CompletedProcess[str]:
return subprocess.run(
["docker", "image", "inspect", "-f", "{{json .RootFS.Layers}}", ref],
capture_output=True,
text=True,
)
p = inspect_layers(image)
if p.returncode != 0:
# Docker Desktop occasionally ends up in a state where repo:tag exists in `docker images`
# but `docker image inspect repo:tag` fails. Inspecting by content-addressed ID works.
id_p = subprocess.run(
["docker", "images", "--no-trunc", "--format", "{{.ID}}", image],
capture_output=True,
text=True,
)
image_id = (id_p.stdout or "").strip()
if not image_id:
return ""
p = inspect_layers(image_id)
if p.returncode != 0:
return ""
return hashlib.sha256(p.stdout.encode()).hexdigest()
def compute_inputs_hash(
context: Path,
dockerfile: Path,
*,
hash_script: Path,
python_exe: str | None = None,
) -> str:
exe = python_exe or sys.executable
p = subprocess.run(
[exe, str(hash_script), "--context", str(context), "--dockerfile", str(dockerfile)],
capture_output=True,
text=True,
)
if p.returncode != 0:
return ""
return p.stdout.strip()
def local_docker_image_tags() -> set[str]:
p = subprocess.run(
["docker", "images", "--format", "{{.Repository}}:{{.Tag}}"],
capture_output=True,
text=True,
)
if p.returncode != 0:
return set()
return {line.strip() for line in p.stdout.splitlines() if line.strip()}